From 559b741749eb2d2a9a7556241508f3efa0fd3b31 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 16 Feb 2023 00:37:10 +0200 Subject: [PATCH 01/34] added REST method in UserController to get users by query --- .../server/controller/UserController.java | 96 +++++++++++++++++++ .../server/common/data/UserData.java | 38 ++++++++ .../server/common/data/query/KeyFilter.java | 6 ++ .../data/query/StringFilterPredicate.java | 6 ++ 4 files changed, 146 insertions(+) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/UserData.java diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 3e202de938..711c67383d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.http.HttpStatus; @@ -38,7 +39,9 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.UserData; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; @@ -46,6 +49,16 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserSettings; @@ -53,6 +66,7 @@ import org.thingsboard.server.common.data.security.event.UserCredentialsInvalida import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.user.TbUserService; import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -62,8 +76,14 @@ import org.thingsboard.server.service.security.system.SystemSecurityService; import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import static org.thingsboard.server.common.data.StringUtils.isNotEmpty; +import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; +import static org.thingsboard.server.common.data.query.FilterPredicateValue.fromString; +import static org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation.EQUAL; import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID; import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.DEFAULT_DASHBOARD; @@ -106,6 +126,9 @@ public class UserController extends BaseController { private final ApplicationEventPublisher eventPublisher; private final TbUserService tbUserService; + @Autowired + private EntityQueryService entityQueryService; + @ApiOperation(value = "Get User (getUserById)", notes = "Fetch the User object based on the provided User Id. " + "If the user has the authority of 'SYS_ADMIN', the server does not perform additional checks. " + @@ -304,6 +327,66 @@ public class UserController extends BaseController { } } + @ApiOperation(value = "Find users by query (findUsersByQuery)", + notes = "Returns a page of user data owned by tenant or customer." + + PAGE_DATA_PARAMETERS + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/users/find", method = RequestMethod.GET) + @ResponseBody + public PageData findUsersByQuery( + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @ApiParam(value = USER_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = USER_SORT_PROPERTY_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { + SecurityUser securityUser = getCurrentUser(); + + EntityTypeFilter entityFilter = new EntityTypeFilter(); + entityFilter.setEntityType(EntityType.USER); + + EntityDataPageLink pageLink = new EntityDataPageLink(pageSize, page, textSearch, createEntityDataSortOrder(sortProperty, sortOrder)); + + List entityFields = Arrays.asList(new EntityKey(ENTITY_FIELD, "firstName"), + new EntityKey(ENTITY_FIELD, "lastName"), + new EntityKey(ENTITY_FIELD, "email")); + + List keyFilters = new ArrayList<>(); + KeyFilter tenantIdFilter = KeyFilter.builder() + .key(new EntityKey(ENTITY_FIELD, "tenantId")) + .predicate(StringFilterPredicate.builder() + .operation(EQUAL) + .value(fromString(securityUser.getTenantId().getId().toString())).build()) + .build(); + keyFilters.add(tenantIdFilter); + + if (!Authority.TENANT_ADMIN.equals(securityUser.getAuthority())) { + KeyFilter customerIdFilter = KeyFilter.builder() + .key(new EntityKey(ENTITY_FIELD, "customerId")) + .predicate(StringFilterPredicate.builder() + .operation(EQUAL) + .value(fromString(securityUser.getCustomerId().getId().toString())).build()) + .build(); + keyFilters.add(customerIdFilter); + } + + EntityDataQuery query = new EntityDataQuery(entityFilter, pageLink, entityFields, null, keyFilters); + + try { + return entityQueryService.findEntityDataByQuery(securityUser, query).mapData(entityData -> + new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), + entityData.getLatest().get(ENTITY_FIELD).get("email").getValue(), + entityData.getLatest().get(ENTITY_FIELD).get("firstName").getValue(), + entityData.getLatest().get(ENTITY_FIELD).get("lastName").getValue())); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Get Tenant Users (getTenantAdmins)", notes = "Returns a page of users owned by tenant. " + PAGE_DATA_PARAMETERS + SYSTEM_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('SYS_ADMIN')") @@ -434,4 +517,17 @@ public class UserController extends BaseController { SecurityUser currentUser = getCurrentUser(); userSettingsService.deleteUserSettings(currentUser.getTenantId(), currentUser.getId(), Arrays.asList(paths.split(","))); } + + private EntityDataSortOrder createEntityDataSortOrder(String sortProperty, String sortOrder) { + if (isNotEmpty(sortProperty)) { + EntityDataSortOrder entityDataSortOrder = new EntityDataSortOrder(); + entityDataSortOrder.setKey(new EntityKey(ENTITY_FIELD, sortProperty)); + if (isNotEmpty(sortOrder)) { + entityDataSortOrder.setDirection(EntityDataSortOrder.Direction.valueOf(sortOrder)); + } + return entityDataSortOrder; + } else { + return null; + } + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java b/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java new file mode 100644 index 0000000000..fd596ba639 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.UserId; + +@ApiModel +@Data +@AllArgsConstructor +public class UserData { + + @ApiModelProperty(position = 1, value = "User id") + private UserId id; + @ApiModelProperty(position = 2, value = "User email", example = "john@gmail.com") + private String email; + @ApiModelProperty(position = 3, value = "User first name", example = "John") + private String firstName; + @ApiModelProperty(position = 4, value = "User last name", example = "Brown") + private String lastName; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java index ad5d42e31d..c81b2b8db6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java @@ -16,12 +16,18 @@ package org.thingsboard.server.common.data.query; import io.swagger.annotations.ApiModel; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import java.io.Serializable; @ApiModel @Data +@Builder +@AllArgsConstructor +@NoArgsConstructor public class KeyFilter implements Serializable { private EntityKey key; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java index 125ca4665a..f30ec6c9fd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java @@ -15,11 +15,17 @@ */ package org.thingsboard.server.common.data.query; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import javax.validation.Valid; @Data +@Builder +@AllArgsConstructor +@NoArgsConstructor public class StringFilterPredicate implements SimpleKeyFilterPredicate { private StringOperation operation; From 314071b7615a3b7435fdb070fca3f6bbfd212c41 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 13 Feb 2023 14:22:32 +0100 Subject: [PATCH 02/34] SPRING_DATASOURCE_HIKARI_REGISTER_MBEANS parameter added --- application/src/main/resources/thingsboard.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 451377c3b0..ecd624172c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -566,6 +566,7 @@ spring: password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:16}" + registerMbeans: "${SPRING_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # true - enable MBean to diagnose pools state via JMX # Audit log parameters audit-log: From 39070ead536873841e93caecf615da19e2e2875e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 16 Feb 2023 12:23:22 +0200 Subject: [PATCH 03/34] refactoring --- .../server/controller/UserController.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 711c67383d..7e768b747f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -376,15 +376,11 @@ public class UserController extends BaseController { EntityDataQuery query = new EntityDataQuery(entityFilter, pageLink, entityFields, null, keyFilters); - try { - return entityQueryService.findEntityDataByQuery(securityUser, query).mapData(entityData -> - new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), - entityData.getLatest().get(ENTITY_FIELD).get("email").getValue(), - entityData.getLatest().get(ENTITY_FIELD).get("firstName").getValue(), - entityData.getLatest().get(ENTITY_FIELD).get("lastName").getValue())); - } catch (Exception e) { - throw handleException(e); - } + return entityQueryService.findEntityDataByQuery(securityUser, query).mapData(entityData -> + new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), + entityData.getLatest().get(ENTITY_FIELD).get("email").getValue(), + entityData.getLatest().get(ENTITY_FIELD).get("firstName").getValue(), + entityData.getLatest().get(ENTITY_FIELD).get("lastName").getValue())); } @ApiOperation(value = "Get Tenant Users (getTenantAdmins)", From 4470ae9a05079da0e25310b9596b6d706f9df51c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 16 Feb 2023 13:45:01 +0200 Subject: [PATCH 04/34] added tests --- .../controller/BaseUserControllerTest.java | 169 ++++++++++++++++++ .../server/common/data/UserData.java | 3 +- 2 files changed, 171 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index 983e33ed7a..5df1c8acb5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.UserData; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -48,6 +49,7 @@ import org.thingsboard.server.service.mail.TestMailService; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; @@ -61,6 +63,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.SYSTEM_TENANT; public abstract class BaseUserControllerTest extends AbstractControllerTest { private IdComparator idComparator = new IdComparator<>(); + private IdComparator userDataIdComparator = new IdComparator<>(); private CustomerId customerNUULId = (CustomerId) createEntityId_NULL_UUID(new Customer()); @@ -858,6 +861,172 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { Assert.assertEquals(expectedSettings, retrievedSettings); } + @Test + public void shouldFindCustomerUsersByFirstName() throws Exception { + loginSysAdmin(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(tenantId); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + + createUserAndLogin(tenantAdmin, "testPassword1"); + + Customer customer = new Customer(); + customer.setTitle("My customer"); + Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + CustomerId customerId = savedCustomer.getId(); + + Customer customer2 = new Customer(); + customer2.setTitle("My customer2"); + Customer savedCustomer2 = doPost("/api/customer", customer2, Customer.class); + CustomerId customerId2 = savedCustomer2.getId(); + + List customerUsers = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + User user = new User(); + user.setAuthority(Authority.CUSTOMER_USER); + user.setCustomerId(customerId); + user.setFirstName("Name" + i); + user.setLastName("Lastname" + i); + user.setEmail("testCustomer" + i + "@thingsboard.org"); + customerUsers.add(doPost("/api/user", user, User.class)); + } + for (int i = 0; i < 10; i++) { + User user = new User(); + user.setAuthority(Authority.CUSTOMER_USER); + user.setCustomerId(customerId2); + user.setFirstName("SecondCustomerName" + i); + user.setLastName("SecondCustomerLastname" + i); + user.setEmail("SecondCustomerUser" + i + "@thingsboard.org"); + doPost("/api/user", user, User.class); + } + + User user = new User(); + user.setAuthority(Authority.CUSTOMER_USER); + user.setCustomerId(customerId); + user.setEmail("testCustomerUser@thingsboard.org"); + createUserAndLogin(user, "testPassword2"); + + // find user my name + List loadedCustomerUsers = new ArrayList<>(); + PageLink pageLink = new PageLink(10, 0, "Name"); + PageData pageData = null; + do { + pageData = doGetTypedWithPageLink("/api/users/find?", + new TypeReference<>() { + }, pageLink); + loadedCustomerUsers.addAll(pageData.getData()); + if (pageData.hasNext()) { + pageLink = pageLink.nextPageLink(); + } + } while (pageData.hasNext()); + + List customerUserDatas = customerUsers.stream().map(customerUser -> new UserData(customerUser.getId(), + customerUser.getEmail(), customerUser.getFirstName(), customerUser.getLastName())) + .sorted(userDataIdComparator).collect(Collectors.toList()); + loadedCustomerUsers.sort(userDataIdComparator); + + Assert.assertEquals(customerUserDatas, loadedCustomerUsers); + + // find user my full name + loadedCustomerUsers.clear(); + pageLink = new PageLink(10, 0, "Name3"); + pageData = doGetTypedWithPageLink("/api/users/find?", + new TypeReference<>() { + }, pageLink); + loadedCustomerUsers.addAll(pageData.getData()); + Assert.assertEquals(pageData.getData().size(), 1); + Assert.assertEquals(pageData.getData().get(0).getEmail(), "testCustomer3@thingsboard.org"); + + //clear users + loginUser(tenantAdmin.getEmail(), "testPassword1"); + doDelete("/api/customer/" + customerId.getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void shouldFindTenantUsersByLastName() throws Exception { + loginSysAdmin(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(tenantId); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + + createUserAndLogin(tenantAdmin, "testPassword1"); + + Customer customer = new Customer(); + customer.setTitle("My customer"); + Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + CustomerId customerId = savedCustomer.getId(); + + Customer customer2 = new Customer(); + customer2.setTitle("My customer2"); + Customer savedCustomer2 = doPost("/api/customer", customer2, Customer.class); + CustomerId customerId2 = savedCustomer2.getId(); + + List customerUsers = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + User user = new User(); + user.setAuthority(Authority.CUSTOMER_USER); + user.setCustomerId(customerId); + user.setFirstName("Name" + i); + user.setLastName("Lastname" + i); + user.setEmail("testCustomer" + i + "@thingsboard.org"); + customerUsers.add(doPost("/api/user", user, User.class)); + } + for (int i = 0; i < 10; i++) { + User user = new User(); + user.setAuthority(Authority.CUSTOMER_USER); + user.setCustomerId(customerId2); + user.setFirstName("SecondCustomerName" + i); + user.setLastName("SecondCustomerLastname" + i); + user.setEmail("SecondCustomerUser" + i + "@thingsboard.org"); + customerUsers.add(doPost("/api/user", user, User.class)); + } + + // find user my name + List loadedCustomerUsers = new ArrayList<>(); + PageLink pageLink = new PageLink(10, 0, "Name"); + PageData pageData = null; + do { + pageData = doGetTypedWithPageLink("/api/users/find?", + new TypeReference<>() { + }, pageLink); + loadedCustomerUsers.addAll(pageData.getData()); + if (pageData.hasNext()) { + pageLink = pageLink.nextPageLink(); + } + } while (pageData.hasNext()); + + List customerUserDatas = customerUsers.stream().map(customerUser -> new UserData(customerUser.getId(), + customerUser.getEmail(), customerUser.getFirstName(), customerUser.getLastName())) + .sorted(userDataIdComparator).collect(Collectors.toList()); + loadedCustomerUsers.sort(userDataIdComparator); + + Assert.assertEquals(customerUserDatas, loadedCustomerUsers); + + // find user my full name + loadedCustomerUsers.clear(); + pageLink = new PageLink(10, 0, "SecondCustomerLastname3"); + pageData = doGetTypedWithPageLink("/api/users/find?", + new TypeReference<>() { + }, pageLink); + loadedCustomerUsers.addAll(pageData.getData()); + Assert.assertEquals(pageData.getData().size(), 1); + Assert.assertEquals(pageData.getData().get(0).getEmail(), "SecondCustomerUser3@thingsboard.org"); + + //clear users + loginUser(tenantAdmin.getEmail(), "testPassword1"); + doDelete("/api/customer/" + customerId.getId().toString()) + .andExpect(status().isOk()); + } + private User createUser() throws Exception { loginSysAdmin(); String email = "tenant2@thingsboard.org"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java b/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java index fd596ba639..8edb5ec7cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java @@ -19,12 +19,13 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.UserId; @ApiModel @Data @AllArgsConstructor -public class UserData { +public class UserData implements HasId { @ApiModelProperty(position = 1, value = "User id") private UserId id; From a613e1dbc6ed68e6533533561d43201252de50fe Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 16 Feb 2023 17:46:24 +0200 Subject: [PATCH 05/34] refactoring --- .../server/controller/UserController.java | 37 ++----------------- .../controller/BaseUserControllerTest.java | 8 ++-- .../server/common/data/query/KeyFilter.java | 6 --- .../data/query/StringFilterPredicate.java | 6 --- 4 files changed, 8 insertions(+), 49 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 7e768b747f..032d664c1b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -49,16 +49,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityTypeFilter; -import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.query.KeyFilter; -import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserSettings; @@ -76,14 +71,11 @@ import org.thingsboard.server.service.security.system.SystemSecurityService; import javax.servlet.http.HttpServletRequest; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.thingsboard.server.common.data.StringUtils.isNotEmpty; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; -import static org.thingsboard.server.common.data.query.FilterPredicateValue.fromString; -import static org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation.EQUAL; import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID; import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.DEFAULT_DASHBOARD; @@ -328,10 +320,10 @@ public class UserController extends BaseController { } @ApiOperation(value = "Find users by query (findUsersByQuery)", - notes = "Returns a page of user data owned by tenant or customer." + - PAGE_DATA_PARAMETERS + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + notes = "Returns page of user data objects. Search is been executed by email, firstName and " + + "lastName fields. " + PAGE_DATA_PARAMETERS + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/users/find", method = RequestMethod.GET) + @RequestMapping(value = "/users/info", method = RequestMethod.GET) @ResponseBody public PageData findUsersByQuery( @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @@ -348,33 +340,12 @@ public class UserController extends BaseController { EntityTypeFilter entityFilter = new EntityTypeFilter(); entityFilter.setEntityType(EntityType.USER); - EntityDataPageLink pageLink = new EntityDataPageLink(pageSize, page, textSearch, createEntityDataSortOrder(sortProperty, sortOrder)); - List entityFields = Arrays.asList(new EntityKey(ENTITY_FIELD, "firstName"), new EntityKey(ENTITY_FIELD, "lastName"), new EntityKey(ENTITY_FIELD, "email")); - List keyFilters = new ArrayList<>(); - KeyFilter tenantIdFilter = KeyFilter.builder() - .key(new EntityKey(ENTITY_FIELD, "tenantId")) - .predicate(StringFilterPredicate.builder() - .operation(EQUAL) - .value(fromString(securityUser.getTenantId().getId().toString())).build()) - .build(); - keyFilters.add(tenantIdFilter); - - if (!Authority.TENANT_ADMIN.equals(securityUser.getAuthority())) { - KeyFilter customerIdFilter = KeyFilter.builder() - .key(new EntityKey(ENTITY_FIELD, "customerId")) - .predicate(StringFilterPredicate.builder() - .operation(EQUAL) - .value(fromString(securityUser.getCustomerId().getId().toString())).build()) - .build(); - keyFilters.add(customerIdFilter); - } - - EntityDataQuery query = new EntityDataQuery(entityFilter, pageLink, entityFields, null, keyFilters); + EntityDataQuery query = new EntityDataQuery(entityFilter, pageLink, entityFields, null, null); return entityQueryService.findEntityDataByQuery(securityUser, query).mapData(entityData -> new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index 5df1c8acb5..4c7b783dc0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -915,7 +915,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { PageLink pageLink = new PageLink(10, 0, "Name"); PageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/users/find?", + pageData = doGetTypedWithPageLink("/api/users/info?", new TypeReference<>() { }, pageLink); loadedCustomerUsers.addAll(pageData.getData()); @@ -934,7 +934,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { // find user my full name loadedCustomerUsers.clear(); pageLink = new PageLink(10, 0, "Name3"); - pageData = doGetTypedWithPageLink("/api/users/find?", + pageData = doGetTypedWithPageLink("/api/users/info?", new TypeReference<>() { }, pageLink); loadedCustomerUsers.addAll(pageData.getData()); @@ -995,7 +995,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { PageLink pageLink = new PageLink(10, 0, "Name"); PageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/users/find?", + pageData = doGetTypedWithPageLink("/api/users/info?", new TypeReference<>() { }, pageLink); loadedCustomerUsers.addAll(pageData.getData()); @@ -1014,7 +1014,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { // find user my full name loadedCustomerUsers.clear(); pageLink = new PageLink(10, 0, "SecondCustomerLastname3"); - pageData = doGetTypedWithPageLink("/api/users/find?", + pageData = doGetTypedWithPageLink("/api/users/info?", new TypeReference<>() { }, pageLink); loadedCustomerUsers.addAll(pageData.getData()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java index c81b2b8db6..ad5d42e31d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/KeyFilter.java @@ -16,18 +16,12 @@ package org.thingsboard.server.common.data.query; import io.swagger.annotations.ApiModel; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; import java.io.Serializable; @ApiModel @Data -@Builder -@AllArgsConstructor -@NoArgsConstructor public class KeyFilter implements Serializable { private EntityKey key; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java index f30ec6c9fd..125ca4665a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java @@ -15,17 +15,11 @@ */ package org.thingsboard.server.common.data.query; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; import javax.validation.Valid; @Data -@Builder -@AllArgsConstructor -@NoArgsConstructor public class StringFilterPredicate implements SimpleKeyFilterPredicate { private StringOperation operation; From bbbea0fc3158ca093a04966b92117e980b42c9ec Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 17 Feb 2023 00:01:50 +0200 Subject: [PATCH 06/34] added tests, refactoring --- .../server/controller/BaseController.java | 17 + .../server/controller/UserController.java | 25 +- .../controller/BaseUserControllerTest.java | 397 +++++++----------- 3 files changed, 187 insertions(+), 252 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 970100d938..92536b985a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -93,6 +93,8 @@ import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rule.RuleChain; @@ -160,6 +162,8 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.StringUtils.isNotEmpty; +import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; import static org.thingsboard.server.controller.ControllerConstants.INCORRECT_TENANT_ID; import static org.thingsboard.server.controller.UserController.YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -987,4 +991,17 @@ public abstract class BaseController { }, MoreExecutors.directExecutor()); return deferredResult; } + + protected EntityDataSortOrder createEntityDataSortOrder(String sortProperty, String sortOrder) { + if (isNotEmpty(sortProperty)) { + EntityDataSortOrder entityDataSortOrder = new EntityDataSortOrder(); + entityDataSortOrder.setKey(new EntityKey(ENTITY_FIELD, sortProperty)); + if (isNotEmpty(sortOrder)) { + entityDataSortOrder.setDirection(EntityDataSortOrder.Direction.valueOf(sortOrder)); + } + return entityDataSortOrder; + } else { + return null; + } + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 032d664c1b..6177bdc74b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -54,6 +54,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.UserSettings; @@ -73,6 +74,7 @@ import javax.servlet.http.HttpServletRequest; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.thingsboard.server.common.data.StringUtils.isNotEmpty; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @@ -348,10 +350,13 @@ public class UserController extends BaseController { EntityDataQuery query = new EntityDataQuery(entityFilter, pageLink, entityFields, null, null); return entityQueryService.findEntityDataByQuery(securityUser, query).mapData(entityData -> - new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), - entityData.getLatest().get(ENTITY_FIELD).get("email").getValue(), - entityData.getLatest().get(ENTITY_FIELD).get("firstName").getValue(), - entityData.getLatest().get(ENTITY_FIELD).get("lastName").getValue())); + { + Map keyValueMap = entityData.getLatest().get(ENTITY_FIELD); + return new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), + keyValueMap.get("email").getValue(), + keyValueMap.get("firstName").getValue(), + keyValueMap.get("lastName").getValue()); + }); } @ApiOperation(value = "Get Tenant Users (getTenantAdmins)", @@ -485,16 +490,4 @@ public class UserController extends BaseController { userSettingsService.deleteUserSettings(currentUser.getTenantId(), currentUser.getId(), Arrays.asList(paths.split(","))); } - private EntityDataSortOrder createEntityDataSortOrder(String sortProperty, String sortOrder) { - if (isNotEmpty(sortProperty)) { - EntityDataSortOrder entityDataSortOrder = new EntityDataSortOrder(); - entityDataSortOrder.setKey(new EntityKey(ENTITY_FIELD, sortProperty)); - if (isNotEmpty(sortOrder)) { - entityDataSortOrder.setDirection(EntityDataSortOrder.Direction.valueOf(sortOrder)); - } - return entityDataSortOrder; - } else { - return null; - } - } } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index 4c7b783dc0..d78b8b08ef 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Assert; import org.junit.Ignore; @@ -87,21 +88,15 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { public void testSaveUser() throws Exception { loginSysAdmin(); - String email = "tenant2@thingsboard.org"; - User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setTenantId(tenantId); - user.setEmail(email); - user.setFirstName("Joe"); - user.setLastName("Downs"); - + User user = createTenantAdminUser(); + String email = user.getEmail(); Mockito.reset(tbClusterService, auditLogService); User savedUser = doPost("/api/user", user, User.class); Assert.assertNotNull(savedUser); Assert.assertNotNull(savedUser.getId()); Assert.assertTrue(savedUser.getCreatedTime() > 0); - Assert.assertEquals(user.getEmail(), savedUser.getEmail()); + Assert.assertEquals(email, savedUser.getEmail()); User foundUser = doGet("/api/user/" + savedUser.getId().getId().toString(), User.class); Assert.assertEquals(foundUser, savedUser); @@ -156,13 +151,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - String email = "tenant2@thingsboard.org"; - User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setTenantId(tenantId); - user.setEmail(email); - user.setFirstName(StringUtils.randomAlphabetic(300)); - user.setLastName("Downs"); + User user = createTenantAdminUser(StringUtils.randomAlphabetic(300), "Brown"); String msgError = msgErrorFieldLength("first name"); doPost("/api/user", user) .andExpect(status().isBadRequest()) @@ -189,12 +178,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { public void testUpdateUserFromDifferentTenant() throws Exception { loginSysAdmin(); - User tenantAdmin = new User(); - tenantAdmin.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin.setTenantId(tenantId); - tenantAdmin.setEmail("tenant2@thingsboard.org"); - tenantAdmin.setFirstName("Joe"); - tenantAdmin.setLastName("Downs"); + User tenantAdmin = createTenantAdminUser(); tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); loginDifferentTenant(); @@ -214,14 +198,8 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { public void testResetPassword() throws Exception { loginSysAdmin(); - String email = "tenant2@thingsboard.org"; - User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setTenantId(tenantId); - user.setEmail(email); - user.setFirstName("Joe"); - user.setLastName("Downs"); - + User user = createTenantAdminUser(); + String email = user.getEmail(); User savedUser = createUserAndLogin(user, "testPassword1"); resetTokens(); @@ -266,13 +244,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { public void testFindUserById() throws Exception { loginSysAdmin(); - String email = "tenant2@thingsboard.org"; - User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setTenantId(tenantId); - user.setEmail(email); - user.setFirstName("Joe"); - user.setLastName("Downs"); + User user = createTenantAdminUser(); User savedUser = doPost("/api/user", user, User.class); User foundUser = doGet("/api/user/" + savedUser.getId().getId().toString(), User.class); @@ -286,15 +258,12 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - String email = TENANT_ADMIN_EMAIL; User user = new User(); user.setAuthority(Authority.TENANT_ADMIN); user.setTenantId(tenantId); - user.setEmail(email); - user.setFirstName("Joe"); - user.setLastName("Downs"); + user.setEmail(TENANT_ADMIN_EMAIL); - String msgError = "User with email '" + email + "' already present in database"; + String msgError = "User with email '" + TENANT_ADMIN_EMAIL + "' already present in database"; doPost("/api/user", user) .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); @@ -311,12 +280,8 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); String email = "tenant_thingsboard.org"; - User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setTenantId(tenantId); + User user = createTenantAdminUser(); user.setEmail(email); - user.setFirstName("Joe"); - user.setLastName("Downs"); String msgError = "Invalid email address format '" + email + "'"; doPost("/api/user", user) @@ -377,13 +342,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { public void testDeleteUser() throws Exception { loginSysAdmin(); - String email = "tenant2@thingsboard.org"; - User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setTenantId(tenantId); - user.setEmail(email); - user.setFirstName("Joe"); - user.setLastName("Downs"); + User user = createTenantAdminUser(); User savedUser = doPost("/api/user", user, User.class); User foundUser = doGet("/api/user/" + savedUser.getId().getId().toString(), User.class); @@ -565,20 +524,10 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { public void testFindCustomerUsers() throws Exception { loginSysAdmin(); - User tenantAdmin = new User(); - tenantAdmin.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin.setTenantId(tenantId); - tenantAdmin.setEmail("tenant2@thingsboard.org"); - tenantAdmin.setFirstName("Joe"); - tenantAdmin.setLastName("Downs"); - + User tenantAdmin = createTenantAdminUser(); createUserAndLogin(tenantAdmin, "testPassword1"); - Customer customer = new Customer(); - customer.setTitle("My customer"); - Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - - CustomerId customerId = savedCustomer.getId(); + CustomerId customerId = postCustomer(); List customerUsers = new ArrayList<>(); for (int i = 0; i < 56; i++) { @@ -615,47 +564,22 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { public void testFindCustomerUsersByEmail() throws Exception { loginSysAdmin(); - User tenantAdmin = new User(); - tenantAdmin.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin.setTenantId(tenantId); - tenantAdmin.setEmail("tenant2@thingsboard.org"); - tenantAdmin.setFirstName("Joe"); - tenantAdmin.setLastName("Downs"); - + User tenantAdmin = createTenantAdminUser(); createUserAndLogin(tenantAdmin, "testPassword1"); - Customer customer = new Customer(); - customer.setTitle("My customer"); - Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - - CustomerId customerId = savedCustomer.getId(); + CustomerId customerId = postCustomer(); String email1 = "testEmail1"; - List customerUsersEmail1 = new ArrayList<>(); - - for (int i = 0; i < 74; i++) { - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setCustomerId(customerId); - String suffix = StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)); - String email = email1 + suffix + "@thingsboard.org"; - email = i % 2 == 0 ? email.toLowerCase() : email.toUpperCase(); - user.setEmail(email); - customerUsersEmail1.add(doPost("/api/user", user, User.class)); - } - String email2 = "testEmail2"; - List customerUsersEmail2 = new ArrayList<>(); - - for (int i = 0; i < 92; i++) { - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setCustomerId(customerId); - String suffix = StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)); - String email = email2 + suffix + "@thingsboard.org"; - email = i % 2 == 0 ? email.toLowerCase() : email.toUpperCase(); - user.setEmail(email); - customerUsersEmail2.add(doPost("/api/user", user, User.class)); + List customerUsersEmail1 = new ArrayList<>(); + List customerUsersEmail2= new ArrayList<>(); + for (int i = 0; i < 45; i++) { + User customerUser = createCustomerUser( customerId); + customerUser.setEmail(email1 + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org"); + customerUsersEmail1.add(doPost("/api/user", customerUser, User.class)); + + customerUser.setEmail(email2 + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org"); + customerUsersEmail2.add(doPost("/api/user", customerUser, User.class)); } List loadedCustomerUsersEmail1 = new ArrayList<>(); @@ -723,14 +647,18 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { @Test public void testDeleteUserWithDeleteRelationsOk() throws Exception { - UserId userId = createUser().getId(); + loginSysAdmin(); + User tenantAdminUser = createTenantAdminUser(); + UserId userId = doPost("/api/user", tenantAdminUser, User.class).getId(); testEntityDaoWithRelationsOk(tenantId, userId, "/api/user/" + userId); } @Ignore @Test public void testDeleteUserExceptionWithRelationsTransactional() throws Exception { - UserId userId = createUser().getId(); + loginSysAdmin(); + User tenantAdminUser = createTenantAdminUser("Joe", "Downs"); + UserId userId = doPost("/api/user", tenantAdminUser, User.class).getId(); testEntityDaoWithRelationsTransactionalException(userDao, tenantId, userId, "/api/user/" + userId); } @@ -862,180 +790,177 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { } @Test - public void shouldFindCustomerUsersByFirstName() throws Exception { + public void checkCustomerUserDoNotSeeTenantAndOtherCustomerUsers() throws Exception { loginSysAdmin(); - User tenantAdmin = new User(); - tenantAdmin.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin.setTenantId(tenantId); - tenantAdmin.setEmail("tenant2@thingsboard.org"); - tenantAdmin.setFirstName("Joe"); - tenantAdmin.setLastName("Downs"); + User tenantAdmin = createTenantAdminUser("Joe", "Brown"); + createUserAndLogin(tenantAdmin, "testPassword1"); + + CustomerId customerId = postCustomer(); + User user = createCustomerUser("Joe", "Downs", customerId); + doPost("/api/user", user, User.class); + CustomerId customerId2 = postCustomer(); + User user2 = createCustomerUser(customerId2); + createUserAndLogin(user2, "testPassword2"); + + PageLink pageLink = new PageLink(10, 0, "Joe"); + List usersInfo = getUsersInfo(pageLink); + + Assert.assertEquals(usersInfo.size(), 0); + } + + @Test + public void shouldFindCustomerUsersByFirstName() throws Exception { + loginSysAdmin(); + User tenantAdmin = createTenantAdminUser(); createUserAndLogin(tenantAdmin, "testPassword1"); - Customer customer = new Customer(); - customer.setTitle("My customer"); - Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - CustomerId customerId = savedCustomer.getId(); + String searchText = "Philip"; - Customer customer2 = new Customer(); - customer2.setTitle("My customer2"); - Customer savedCustomer2 = doPost("/api/customer", customer2, Customer.class); - CustomerId customerId2 = savedCustomer2.getId(); + CustomerId customerId = postCustomer(); + CustomerId customerId2 = postCustomer(); - List customerUsers = new ArrayList<>(); + List customerUsersContainingWord = new ArrayList<>(); for (int i = 0; i < 10; i++) { - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setCustomerId(customerId); - user.setFirstName("Name" + i); - user.setLastName("Lastname" + i); - user.setEmail("testCustomer" + i + "@thingsboard.org"); - customerUsers.add(doPost("/api/user", user, User.class)); - } - for (int i = 0; i < 10; i++) { - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setCustomerId(customerId2); - user.setFirstName("SecondCustomerName" + i); - user.setLastName("SecondCustomerLastname" + i); - user.setEmail("SecondCustomerUser" + i + "@thingsboard.org"); - doPost("/api/user", user, User.class); + String suffix = StringUtils.randomAlphabetic((int) (5 + Math.random() * 10)); + + customerUsersContainingWord.add(doPost("/api/user", createCustomerUser(searchText + i, "Last" + i, customerId), User.class)); + customerUsersContainingWord.add(doPost("/api/user", createCustomerUser(null, null, searchText + suffix + "@thingsboard.org", customerId), User.class)); + doPost("/api/user", createCustomerUser(null, null, customerId), User.class); + + doPost("/api/user", createCustomerUser(searchText + i, "Last" + i, customerId2), User.class); } - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setCustomerId(customerId); - user.setEmail("testCustomerUser@thingsboard.org"); - createUserAndLogin(user, "testPassword2"); + createUserAndLogin(createCustomerUser(customerId), "testPassword2"); - // find user my name - List loadedCustomerUsers = new ArrayList<>(); - PageLink pageLink = new PageLink(10, 0, "Name"); - PageData pageData = null; - do { - pageData = doGetTypedWithPageLink("/api/users/info?", - new TypeReference<>() { - }, pageLink); - loadedCustomerUsers.addAll(pageData.getData()); - if (pageData.hasNext()) { - pageLink = pageLink.nextPageLink(); - } - } while (pageData.hasNext()); + // find users by search text + PageLink pageLink = new PageLink(10, 0, searchText); + List usersInfo = getUsersInfo(pageLink); - List customerUserDatas = customerUsers.stream().map(customerUser -> new UserData(customerUser.getId(), - customerUser.getEmail(), customerUser.getFirstName(), customerUser.getLastName())) + List expectedUserInfos = customerUsersContainingWord.stream().map(customerUser -> new UserData(customerUser.getId(), + customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), + customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); - loadedCustomerUsers.sort(userDataIdComparator); + usersInfo.sort(userDataIdComparator); - Assert.assertEquals(customerUserDatas, loadedCustomerUsers); + Assert.assertEquals(expectedUserInfos, usersInfo); - // find user my full name - loadedCustomerUsers.clear(); - pageLink = new PageLink(10, 0, "Name3"); - pageData = doGetTypedWithPageLink("/api/users/info?", - new TypeReference<>() { - }, pageLink); - loadedCustomerUsers.addAll(pageData.getData()); - Assert.assertEquals(pageData.getData().size(), 1); - Assert.assertEquals(pageData.getData().get(0).getEmail(), "testCustomer3@thingsboard.org"); + // find user by full name + pageLink = new PageLink(10, 0, searchText + "5"); + usersInfo = getUsersInfo(pageLink); + Assert.assertEquals(1, usersInfo.size()); //clear users loginUser(tenantAdmin.getEmail(), "testPassword1"); doDelete("/api/customer/" + customerId.getId().toString()) .andExpect(status().isOk()); + doDelete("/api/customer/" + customerId2.getId().toString()) + .andExpect(status().isOk()); } @Test - public void shouldFindTenantUsersByLastName() throws Exception { + public void shouldFindTenantUsersByTextFromLastName() throws Exception { loginSysAdmin(); - User tenantAdmin = new User(); - tenantAdmin.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin.setTenantId(tenantId); - tenantAdmin.setEmail("tenant2@thingsboard.org"); - tenantAdmin.setFirstName("Joe"); - tenantAdmin.setLastName("Downs"); - + User tenantAdmin = createTenantAdminUser(); createUserAndLogin(tenantAdmin, "testPassword1"); + CustomerId customerId = postCustomer(); + CustomerId customerId2 = postCustomer(); - Customer customer = new Customer(); - customer.setTitle("My customer"); - Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - CustomerId customerId = savedCustomer.getId(); - - Customer customer2 = new Customer(); - customer2.setTitle("My customer2"); - Customer savedCustomer2 = doPost("/api/customer", customer2, Customer.class); - CustomerId customerId2 = savedCustomer2.getId(); + String searchText = "Brown"; - List customerUsers = new ArrayList<>(); + List usersContainingWord = new ArrayList<>(); for (int i = 0; i < 10; i++) { - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setCustomerId(customerId); - user.setFirstName("Name" + i); - user.setLastName("Lastname" + i); - user.setEmail("testCustomer" + i + "@thingsboard.org"); - customerUsers.add(doPost("/api/user", user, User.class)); - } - for (int i = 0; i < 10; i++) { - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setCustomerId(customerId2); - user.setFirstName("SecondCustomerName" + i); - user.setLastName("SecondCustomerLastname" + i); - user.setEmail("SecondCustomerUser" + i + "@thingsboard.org"); - customerUsers.add(doPost("/api/user", user, User.class)); + String suffix = StringUtils.randomAlphabetic((int) (5 + Math.random() * 10)); + usersContainingWord.add(doPost("/api/user", createCustomerUser("First" + i, searchText + i, customerId), User.class)); + usersContainingWord.add(doPost("/api/user", createCustomerUser(null, null, searchText + suffix + "@thingsboard.org", customerId), User.class)); + + suffix = StringUtils.randomAlphabetic((int) (5 + Math.random() * 10)); + usersContainingWord.add(doPost("/api/user", createCustomerUser("First" + i, searchText + i, customerId2), User.class)); + usersContainingWord.add(doPost("/api/user", createCustomerUser(null, null, searchText + suffix + "@thingsboard.org", customerId2), User.class)); } - // find user my name - List loadedCustomerUsers = new ArrayList<>(); - PageLink pageLink = new PageLink(10, 0, "Name"); - PageData pageData = null; - do { - pageData = doGetTypedWithPageLink("/api/users/info?", - new TypeReference<>() { - }, pageLink); - loadedCustomerUsers.addAll(pageData.getData()); - if (pageData.hasNext()) { - pageLink = pageLink.nextPageLink(); - } - } while (pageData.hasNext()); + loginDifferentTenant(); + CustomerId customerId3 = postCustomer(); + doPost("/api/user", createCustomerUser("Jane", searchText, customerId3), User.class); - List customerUserDatas = customerUsers.stream().map(customerUser -> new UserData(customerUser.getId(), - customerUser.getEmail(), customerUser.getFirstName(), customerUser.getLastName())) + // find users by search text + loginUser(tenantAdmin.getEmail(), "testPassword1"); + PageLink pageLink = new PageLink(10, 0, searchText); + List usersInfo = getUsersInfo(pageLink); + + List expectedUserInfos = usersContainingWord.stream().map(customerUser -> new UserData(customerUser.getId(), + customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), + customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); - loadedCustomerUsers.sort(userDataIdComparator); + usersInfo.sort(userDataIdComparator); - Assert.assertEquals(customerUserDatas, loadedCustomerUsers); + Assert.assertEquals(expectedUserInfos, usersInfo); - // find user my full name - loadedCustomerUsers.clear(); - pageLink = new PageLink(10, 0, "SecondCustomerLastname3"); - pageData = doGetTypedWithPageLink("/api/users/info?", - new TypeReference<>() { - }, pageLink); - loadedCustomerUsers.addAll(pageData.getData()); - Assert.assertEquals(pageData.getData().size(), 1); - Assert.assertEquals(pageData.getData().get(0).getEmail(), "SecondCustomerUser3@thingsboard.org"); + // find user by full last name + pageLink = new PageLink(10, 0, searchText + "3"); + usersInfo = getUsersInfo(pageLink); + Assert.assertEquals(2, usersInfo.size()); //clear users - loginUser(tenantAdmin.getEmail(), "testPassword1"); doDelete("/api/customer/" + customerId.getId().toString()) .andExpect(status().isOk()); + doDelete("/api/customer/" + customerId2.getId().toString()) + .andExpect(status().isOk()); } - private User createUser() throws Exception { - loginSysAdmin(); - String email = "tenant2@thingsboard.org"; + private CustomerId postCustomer() { + Customer customer = new Customer(); + customer.setTitle(StringUtils.randomAlphabetic(9)); + Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + return savedCustomer.getId(); + } + + private static User createCustomerUser(CustomerId customerId) { + return createCustomerUser(null, null, customerId); + } + private static User createCustomerUser(String firstName, String lastName, CustomerId customerId) { + String suffix = StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)); + return createCustomerUser(firstName, lastName, "testMail" + suffix + "@thingsboard.org", customerId); + } + + private static User createCustomerUser(String firstName, String lastName, String email, CustomerId customerId) { User user = new User(); - user.setAuthority(Authority.TENANT_ADMIN); - user.setTenantId(tenantId); + user.setAuthority(Authority.CUSTOMER_USER); + user.setFirstName(firstName); + user.setLastName(lastName); + user.setCustomerId(customerId); user.setEmail(email); - user.setFirstName("Joe"); - user.setLastName("Downs"); - return doPost("/api/user", user, User.class); + return user; + } + + private User createTenantAdminUser() { + return createTenantAdminUser(null, null); + } + private User createTenantAdminUser(String firstName, String lastName) { + String suffix = StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(tenantId); + tenantAdmin.setEmail("testEmail" + suffix + "@thingsbord.org"); + tenantAdmin.setFirstName(firstName); + tenantAdmin.setLastName(lastName); + return tenantAdmin; } + + private List getUsersInfo(PageLink pageLink) throws Exception { + List loadedCustomerUsers = new ArrayList<>(); + PageData pageData = null; + do { + pageData = doGetTypedWithPageLink("/api/users/info?", new TypeReference<>() {}, pageLink); + loadedCustomerUsers.addAll(pageData.getData()); + if (pageData.hasNext()) { + pageLink = pageLink.nextPageLink(); + } + } while (pageData.hasNext()); + return loadedCustomerUsers; + } + } From a111473a8f0896e84f19e27c1268a395857350e6 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 17 Feb 2023 00:12:11 +0200 Subject: [PATCH 07/34] minor improvement --- .../server/controller/BaseUserControllerTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index d78b8b08ef..3efecf3ca2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -811,7 +811,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { } @Test - public void shouldFindCustomerUsersByFirstName() throws Exception { + public void shouldFindCustomerUsersBySearchText() throws Exception { loginSysAdmin(); User tenantAdmin = createTenantAdminUser(); createUserAndLogin(tenantAdmin, "testPassword1"); @@ -829,7 +829,9 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { customerUsersContainingWord.add(doPost("/api/user", createCustomerUser(null, null, searchText + suffix + "@thingsboard.org", customerId), User.class)); doPost("/api/user", createCustomerUser(null, null, customerId), User.class); + suffix = StringUtils.randomAlphabetic((int) (5 + Math.random() * 10)); doPost("/api/user", createCustomerUser(searchText + i, "Last" + i, customerId2), User.class); + doPost("/api/user", createCustomerUser(null, null, searchText + suffix + "@thingsboard.org", customerId2), User.class); } createUserAndLogin(createCustomerUser(customerId), "testPassword2"); @@ -860,7 +862,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { } @Test - public void shouldFindTenantUsersByTextFromLastName() throws Exception { + public void shouldFindTenantUsersBySearchText() throws Exception { loginSysAdmin(); User tenantAdmin = createTenantAdminUser(); @@ -875,6 +877,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { String suffix = StringUtils.randomAlphabetic((int) (5 + Math.random() * 10)); usersContainingWord.add(doPost("/api/user", createCustomerUser("First" + i, searchText + i, customerId), User.class)); usersContainingWord.add(doPost("/api/user", createCustomerUser(null, null, searchText + suffix + "@thingsboard.org", customerId), User.class)); + doPost("/api/user", createCustomerUser(null, null, customerId), User.class); suffix = StringUtils.randomAlphabetic((int) (5 + Math.random() * 10)); usersContainingWord.add(doPost("/api/user", createCustomerUser("First" + i, searchText + i, customerId2), User.class)); From 5dec719947d586674ae33f99a1a787943a6ebc66 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 17 Feb 2023 10:45:42 +0200 Subject: [PATCH 08/34] minor improvements to tests --- .../server/controller/UserController.java | 8 ++--- .../controller/BaseUserControllerTest.java | 30 ++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index 6177bdc74b..ef6f7b4075 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -351,11 +351,11 @@ public class UserController extends BaseController { return entityQueryService.findEntityDataByQuery(securityUser, query).mapData(entityData -> { - Map keyValueMap = entityData.getLatest().get(ENTITY_FIELD); + Map fieldValues = entityData.getLatest().get(ENTITY_FIELD); return new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), - keyValueMap.get("email").getValue(), - keyValueMap.get("firstName").getValue(), - keyValueMap.get("lastName").getValue()); + fieldValues.get("email").getValue(), + fieldValues.get("firstName").getValue(), + fieldValues.get("lastName").getValue()); }); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index 3efecf3ca2..b677a47b13 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -790,24 +790,40 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { } @Test - public void checkCustomerUserDoNotSeeTenantAndOtherCustomerUsers() throws Exception { + public void checkCustomerUserDoNotSeeTenantUsersOtherTenantUsersOtherCustomerUsers() throws Exception { loginSysAdmin(); + String searchText = "Joe"; - User tenantAdmin = createTenantAdminUser("Joe", "Brown"); + loginDifferentTenant(); + CustomerId customerId1 = postCustomer(); + doPost("/api/user", createCustomerUser(searchText, "Ress", customerId1), User.class); + + loginSysAdmin(); + User tenantAdmin = createTenantAdminUser(searchText, "Brown"); createUserAndLogin(tenantAdmin, "testPassword1"); - CustomerId customerId = postCustomer(); - User user = createCustomerUser("Joe", "Downs", customerId); + CustomerId customerId2 = postCustomer(); + User user = createCustomerUser(searchText, "Downs", customerId2); doPost("/api/user", user, User.class); - CustomerId customerId2 = postCustomer(); - User user2 = createCustomerUser(customerId2); + CustomerId customerId3 = postCustomer(); + User user2 = createCustomerUser(customerId3); createUserAndLogin(user2, "testPassword2"); - PageLink pageLink = new PageLink(10, 0, "Joe"); + PageLink pageLink = new PageLink(10, 0, searchText); List usersInfo = getUsersInfo(pageLink); Assert.assertEquals(usersInfo.size(), 0); + + //clear users + loginDifferentTenant(); + doDelete("/api/customer/" + customerId1.getId().toString()) + .andExpect(status().isOk()); + loginUser(tenantAdmin.getEmail(), "testPassword1"); + doDelete("/api/customer/" + customerId2.getId().toString()) + .andExpect(status().isOk()); + doDelete("/api/customer/" + customerId3.getId().toString()) + .andExpect(status().isOk()); } @Test From b261cb2ab1b7af75c1d47ab51ace645eb8e29be6 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 17 Feb 2023 11:12:46 +0200 Subject: [PATCH 09/34] renamed UserData to UserEmailInfo --- .../server/controller/UserController.java | 8 +++---- .../controller/BaseUserControllerTest.java | 21 +++++++++---------- .../{UserData.java => UserEmailInfo.java} | 2 +- 3 files changed, 14 insertions(+), 17 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/{UserData.java => UserEmailInfo.java} (96%) diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index ef6f7b4075..e57bb73262 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -41,7 +41,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.UserData; +import org.thingsboard.server.common.data.UserEmailInfo; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; @@ -51,7 +51,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.TsValue; @@ -76,7 +75,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import static org.thingsboard.server.common.data.StringUtils.isNotEmpty; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID; import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID_PARAM_DESCRIPTION; @@ -327,7 +325,7 @@ public class UserController extends BaseController { @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/users/info", method = RequestMethod.GET) @ResponseBody - public PageData findUsersByQuery( + public PageData findUsersByQuery( @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -352,7 +350,7 @@ public class UserController extends BaseController { return entityQueryService.findEntityDataByQuery(securityUser, query).mapData(entityData -> { Map fieldValues = entityData.getLatest().get(ENTITY_FIELD); - return new UserData(UserId.fromString(entityData.getEntityId().getId().toString()), + return new UserEmailInfo(UserId.fromString(entityData.getEntityId().getId().toString()), fieldValues.get("email").getValue(), fieldValues.get("firstName").getValue(), fieldValues.get("lastName").getValue()); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index b677a47b13..f289c8066f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Assert; import org.junit.Ignore; @@ -35,7 +34,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.UserData; +import org.thingsboard.server.common.data.UserEmailInfo; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -64,7 +63,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.SYSTEM_TENANT; public abstract class BaseUserControllerTest extends AbstractControllerTest { private IdComparator idComparator = new IdComparator<>(); - private IdComparator userDataIdComparator = new IdComparator<>(); + private IdComparator userDataIdComparator = new IdComparator<>(); private CustomerId customerNUULId = (CustomerId) createEntityId_NULL_UUID(new Customer()); @@ -811,7 +810,7 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { createUserAndLogin(user2, "testPassword2"); PageLink pageLink = new PageLink(10, 0, searchText); - List usersInfo = getUsersInfo(pageLink); + List usersInfo = getUsersInfo(pageLink); Assert.assertEquals(usersInfo.size(), 0); @@ -854,9 +853,9 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { // find users by search text PageLink pageLink = new PageLink(10, 0, searchText); - List usersInfo = getUsersInfo(pageLink); + List usersInfo = getUsersInfo(pageLink); - List expectedUserInfos = customerUsersContainingWord.stream().map(customerUser -> new UserData(customerUser.getId(), + List expectedUserInfos = customerUsersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(), customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); @@ -907,9 +906,9 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { // find users by search text loginUser(tenantAdmin.getEmail(), "testPassword1"); PageLink pageLink = new PageLink(10, 0, searchText); - List usersInfo = getUsersInfo(pageLink); + List usersInfo = getUsersInfo(pageLink); - List expectedUserInfos = usersContainingWord.stream().map(customerUser -> new UserData(customerUser.getId(), + List expectedUserInfos = usersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(), customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); @@ -969,9 +968,9 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { return tenantAdmin; } - private List getUsersInfo(PageLink pageLink) throws Exception { - List loadedCustomerUsers = new ArrayList<>(); - PageData pageData = null; + private List getUsersInfo(PageLink pageLink) throws Exception { + List loadedCustomerUsers = new ArrayList<>(); + PageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/users/info?", new TypeReference<>() {}, pageLink); loadedCustomerUsers.addAll(pageData.getData()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java b/common/data/src/main/java/org/thingsboard/server/common/data/UserEmailInfo.java similarity index 96% rename from common/data/src/main/java/org/thingsboard/server/common/data/UserData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/UserEmailInfo.java index 8edb5ec7cf..f1eb95c805 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/UserData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/UserEmailInfo.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.UserId; @ApiModel @Data @AllArgsConstructor -public class UserData implements HasId { +public class UserEmailInfo implements HasId { @ApiModelProperty(position = 1, value = "User id") private UserId id; From 935f5b806d426a8830801ed988f2d1b21056d779 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Mon, 20 Feb 2023 17:01:42 +0200 Subject: [PATCH 10/34] fix issue with remove latest when interval is set to from x to x + 1 where x value of ts entry --- .../thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index afffdbb0cc..b58f0dc7c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -228,7 +228,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme long ts = latest.getTs(); ListenableFuture removedLatestFuture; - if (ts > query.getStartTs() && ts <= query.getEndTs()) { + if (ts >= query.getStartTs() && ts < query.getEndTs()) { TsKvLatestEntity latestEntity = new TsKvLatestEntity(); latestEntity.setEntityId(entityId.getId()); latestEntity.setKey(getOrSaveKeyId(query.getKey())); From 9bbb88ab0fa244c2a84b6deffc9116b2fa8642c5 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 21 Feb 2023 16:50:09 +0200 Subject: [PATCH 11/34] Add functionality to support multiple certificate entries in pem file --- .../credentials/CertPemCredentials.java | 46 +++++--- .../credentials/CertPemCredentialsTest.java | 58 ++++++++++ .../src/test/resources/pem/empty.pem | 0 .../src/test/resources/pem/tb-cloud-chain.pem | 103 ++++++++++++++++++ .../src/test/resources/pem/tb-cloud.pem | 32 ++++++ 5 files changed, 225 insertions(+), 14 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/empty.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 48d71873eb..480c4a0689 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -57,6 +57,10 @@ import java.security.cert.X509Certificate; import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.RSAPrivateCrtKeySpec; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -104,7 +108,7 @@ public class CertPemCredentials implements ClientCredentials { } private KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { - X509Certificate certHolder = readCertFile(cert); + List certHolders = readCertFile(cert); Object keyObject = readPrivateKeyFile(privateKey); char[] passwordCharArray = "".toCharArray(); if (!StringUtils.isEmpty(password)) { @@ -129,43 +133,57 @@ public class CertPemCredentials implements ClientCredentials { KeyStore clientKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); clientKeyStore.load(null, null); - clientKeyStore.setCertificateEntry("cert", certHolder); + for (X509Certificate certHolder : certHolders) { + clientKeyStore.setCertificateEntry(createCertEntryAlias("cert-", certHolder), certHolder); + } clientKeyStore.setKeyEntry("private-key", privateKey, passwordCharArray, - new Certificate[]{certHolder}); - + certHolders.toArray(new Certificate[]{})); KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keyManagerFactory.init(clientKeyStore, passwordCharArray); return keyManagerFactory; } protected TrustManagerFactory createAndInitTrustManagerFactory() throws Exception { - X509Certificate caCertHolder; - caCertHolder = readCertFile(caCert); + List caCertHolders = readCertFile(caCert); KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); - caKeyStore.setCertificateEntry("caCert-cert", caCertHolder); + for (X509Certificate caCertHolder : caCertHolders) { + caKeyStore.setCertificateEntry(createCertEntryAlias("caCert-cert-", caCertHolder), caCertHolder); + } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); trustManagerFactory.init(caKeyStore); return trustManagerFactory; } - private X509Certificate readCertFile(String fileContent) throws Exception { - X509Certificate certificate = null; - if (fileContent != null && !fileContent.trim().isEmpty()) { - fileContent = fileContent.replace("-----BEGIN CERTIFICATE-----", "") + private String createCertEntryAlias(String prefix, X509Certificate certificate) { + return prefix + "-" + certificate.getSubjectDN().getName(); + } + + List readCertFile(String fileContent) throws Exception { + if (fileContent == null || fileContent.trim().isEmpty()) { + return Collections.emptyList(); + } + + List certificates = new ArrayList<>(); + String[] pems = fileContent.trim().split("-----END CERTIFICATE-----"); + for (String pem : pems) { + if (pem.trim().isEmpty()) { + continue; + } + pem = pem.replace("-----BEGIN CERTIFICATE-----", "") .replace("-----END CERTIFICATE-----", "") .replaceAll("\\s", ""); - byte[] decoded = Base64.decodeBase64(fileContent); + byte[] decoded = Base64.decodeBase64(pem); CertificateFactory certFactory = CertificateFactory.getInstance("X.509"); try (InputStream inStream = new ByteArrayInputStream(decoded)) { - certificate = (X509Certificate) certFactory.generateCertificate(inStream); + certificates.add((X509Certificate) certFactory.generateCertificate(inStream)); } } - return certificate; + return certificates; } private PrivateKey readPrivateKeyFile(String fileContent) throws Exception { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java new file mode 100644 index 0000000000..e8126cade5 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -0,0 +1,58 @@ +package org.thingsboard.rule.engine.credentials; + +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.security.cert.X509Certificate; +import java.util.List; + +public class CertPemCredentialsTest { + + private final CertPemCredentials credentials = new CertPemCredentials(); + + @Test + public void testChainOfCertificates() throws Exception { + String fileContent = fileContent("pem/tb-cloud-chain.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(4, x509Certificates.size()); + Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + x509Certificates.get(0).getSubjectDN().getName()); + Assert.assertEquals("CN=Sectigo ECC Organization Validation Secure Server CA, O=Sectigo Limited, L=Salford, ST=Greater Manchester, C=GB", + x509Certificates.get(1).getSubjectDN().getName()); + Assert.assertEquals("CN=USERTrust ECC Certification Authority, O=The USERTRUST Network, L=Jersey City, ST=New Jersey, C=US", + x509Certificates.get(2).getSubjectDN().getName()); + Assert.assertEquals("CN=AAA Certificate Services, O=Comodo CA Limited, L=Salford, ST=Greater Manchester, C=GB", + x509Certificates.get(3).getSubjectDN().getName()); + } + + @Test + public void testSingleCertificate() throws Exception { + String fileContent = fileContent("pem/tb-cloud.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(1, x509Certificates.size()); + Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + x509Certificates.get(0).getSubjectDN().getName()); + } + + @Test + public void testEmptyFileContent() throws Exception { + String fileContent = fileContent("pem/empty.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(0, x509Certificates.size()); + } + + private String fileContent(String fileName) throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + File file = new File(classLoader.getResource(fileName).getFile()); + return FileUtils.readFileToString(file, "UTF-8"); + } +} \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/empty.pem b/rule-engine/rule-engine-components/src/test/resources/pem/empty.pem new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem new file mode 100644 index 0000000000..23049273c7 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem @@ -0,0 +1,103 @@ +-----BEGIN CERTIFICATE----- +MIIFejCCBSCgAwIBAgIQT2YV5NVp2PAY1O5rxMlj6DAKBggqhkjOPQQDAjCBlTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRT +ZWN0aWdvIEVDQyBPcmdhbml6YXRpb24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVy +IENBMB4XDTIxMTAwNTAwMDAwMFoXDTIyMTAwNTIzNTk1OVowWjELMAkGA1UEBhMC +VVMxETAPBgNVBAgTCE5ldyBZb3JrMRowGAYDVQQKExFUaGluZ3NCb2FyZCwgSW5j +LjEcMBoGA1UEAwwTKi50aGluZ3Nib2FyZC5jbG91ZDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABNkK/UerQZXPP0H/Tl8YhRPlzW85yTAcQ5hXhs2fyXn7Bdj4EueQuZrv +Pw98xwHJr87jslFbS/WiSdtBYPvjsUyXqh7aMvOcEhSgEOWDmtoj3P1Xk1hNLb6m +xAQfFL8cZ6OCA20wggNpMB8GA1UdIwQYMBaAFE1K78RGsxKtT06asVniUasIEHgI +MB0GA1UdDgQWBBRr4HG23dsao68r9r7obGxB70ptBzAOBgNVHQ8BAf8EBAMCB4Aw +DAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwSgYD +VR0gBEMwQTA1BgwrBgEEAbIxAQIBAwQwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9z +ZWN0aWdvLmNvbS9DUFMwCAYGZ4EMAQICMFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6 +Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb0VDQ09yZ2FuaXphdGlvblZhbGlkYXRp +b25TZWN1cmVTZXJ2ZXJDQS5jcmwwgYoGCCsGAQUFBwEBBH4wfDBVBggrBgEFBQcw +AoZJaHR0cDovL2NydC5zZWN0aWdvLmNvbS9TZWN0aWdvRUNDT3JnYW5pemF0aW9u +VmFsaWRhdGlvblNlY3VyZVNlcnZlckNBLmNydDAjBggrBgEFBQcwAYYXaHR0cDov +L29jc3Auc2VjdGlnby5jb20wMQYDVR0RBCowKIITKi50aGluZ3Nib2FyZC5jbG91 +ZIIRdGhpbmdzYm9hcmQuY2xvdWQwggGABgorBgEEAdZ5AgQCBIIBcASCAWwBagB2 +AEalVet1+pEgMLWiiWn0830RLEF0vv1JuIWr8vxw/m1HAAABfFDbhtMAAAQDAEcw +RQIhAKNykhkQTngK0yOcYHGHUQSy6JmJYl+5nc1qELirPHwAAiBgV2Db5ZFHNvzn +zp9Ob/OG0o36z6rcilbLI/daZwnyewB3AEHIyrHfIkZKEMahOglCh15OMYsbA+vr +S8do8JBilgb2AAABfFDbhqYAAAQDAEgwRgIhALFvTbapKhO7DPrF6KtE9sjFMMth +qjqaeaHYN6JGnUAIAiEApEW+rxlzxH1+qEwJrFyQLr5rSKTuEoSjv3hbrzb9GQ4A +dwApeb7wnjk5IfBWc59jpXflvld9nGAK+PlNXSZcJV3HhAAAAXxQ24ZnAAAEAwBI +MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL +1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ +GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR +13sr0/ED+2PUom2dnfg= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDrjCCAzOgAwIBAgIQNb50Y4yz6d4oBXC3l4CzZzAKBggqhkjOPQQDAzCBiDEL +MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl +eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT +JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTgxMTAy +MDAwMDAwWhcNMzAxMjMxMjM1OTU5WjCBlTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEYMBYGA1UEChMP +U2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRTZWN0aWdvIEVDQyBPcmdhbml6YXRp +b24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVyIENBMFkwEwYHKoZIzj0CAQYIKoZI +zj0DAQcDQgAEnI5cCmFvoVij0NXO+vxE+f+6Bh57FhpyH0LTCrJmzfsPSXIhTSex +r92HOlz+aHqoGE0vSe/CSwLFoWcZ8W1jOaOCAW4wggFqMB8GA1UdIwQYMBaAFDrh +CYbUzxnClnZ0SXbc4DXGY2OaMB0GA1UdDgQWBBRNSu/ERrMSrU9OmrFZ4lGrCBB4 +CDAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwGwYDVR0gBBQwEjAGBgRVHSAAMAgGBmeBDAEC +AjBQBgNVHR8ESTBHMEWgQ6BBhj9odHRwOi8vY3JsLnVzZXJ0cnVzdC5jb20vVVNF +UlRydXN0RUNDQ2VydGlmaWNhdGlvbkF1dGhvcml0eS5jcmwwdgYIKwYBBQUHAQEE +ajBoMD8GCCsGAQUFBzAChjNodHRwOi8vY3J0LnVzZXJ0cnVzdC5jb20vVVNFUlRy +dXN0RUNDQWRkVHJ1c3RDQS5jcnQwJQYIKwYBBQUHMAGGGWh0dHA6Ly9vY3NwLnVz +ZXJ0cnVzdC5jb20wCgYIKoZIzj0EAwMDaQAwZgIxAOk//uo7i/MoeKdcyeqvjOXs +BJFGLI+1i0d+Tty7zEnn2w4DNS21TK8wmY3Kjm3EmQIxAPI1qHM/I+OS+hx0OZhG +fDoNifTe/GxgWZ1gOYQKzn6lwP0yGKlrP+7vrVC8IczJ4A== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIID0zCCArugAwIBAgIQVmcdBOpPmUxvEIFHWdJ1lDANBgkqhkiG9w0BAQwFADB7 +MQswCQYDVQQGEwJHQjEbMBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYD +VQQHDAdTYWxmb3JkMRowGAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UE +AwwYQUFBIENlcnRpZmljYXRlIFNlcnZpY2VzMB4XDTE5MDMxMjAwMDAwMFoXDTI4 +MTIzMTIzNTk1OVowgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpOZXcgSmVyc2V5 +MRQwEgYDVQQHEwtKZXJzZXkgQ2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBO +ZXR3b3JrMS4wLAYDVQQDEyVVU0VSVHJ1c3QgRUNDIENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEGqxUWqn5aCPnetUkb1PGWthL +q8bVttHmc3Gu3ZzWDGH926CJA7gFFOxXzu5dP+Ihs8731Ip54KODfi2X0GHE8Znc +JZFjq38wo7Rw4sehM5zzvy5cU7Ffs30yf4o043l5o4HyMIHvMB8GA1UdIwQYMBaA +FKARCiM+lvEH7OKvKe+CpX/QMKS0MB0GA1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1 +xmNjmjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zARBgNVHSAECjAI +MAYGBFUdIAAwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5j +b20vQUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNAYIKwYBBQUHAQEEKDAmMCQG +CCsGAQUFBzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQEM +BQADggEBABns652JLCALBIAdGN5CmXKZFjK9Dpx1WywV4ilAbe7/ctvbq5AfjJXy +ij0IckKJUAfiORVsAYfZFhr1wHUrxeZWEQff2Ji8fJ8ZOd+LygBkc7xGEJuTI42+ +FsMuCIKchjN0djsoTI0DQoWz4rIjQtUfenVqGtF8qmchxDM6OW1TyaLtYiKou+JV +bJlsQ2uRl9EMC5MCHdK8aXdJ5htN978UeAOwproLtOGFfy/cQjutdAFI3tZs4RmY +CV4Ks2dH/hzg1cEo70qLRDEmBDeNiXQ2Lu+lIg+DdEmSx/cQwgwp+7e9un/jX9Wf +8qn0dNW44bOwgeThpWOjzOoEeJBuv/c= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- + diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem new file mode 100644 index 0000000000..d75be0bf10 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFejCCBSCgAwIBAgIQT2YV5NVp2PAY1O5rxMlj6DAKBggqhkjOPQQDAjCBlTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRT +ZWN0aWdvIEVDQyBPcmdhbml6YXRpb24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVy +IENBMB4XDTIxMTAwNTAwMDAwMFoXDTIyMTAwNTIzNTk1OVowWjELMAkGA1UEBhMC +VVMxETAPBgNVBAgTCE5ldyBZb3JrMRowGAYDVQQKExFUaGluZ3NCb2FyZCwgSW5j +LjEcMBoGA1UEAwwTKi50aGluZ3Nib2FyZC5jbG91ZDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABNkK/UerQZXPP0H/Tl8YhRPlzW85yTAcQ5hXhs2fyXn7Bdj4EueQuZrv +Pw98xwHJr87jslFbS/WiSdtBYPvjsUyXqh7aMvOcEhSgEOWDmtoj3P1Xk1hNLb6m +xAQfFL8cZ6OCA20wggNpMB8GA1UdIwQYMBaAFE1K78RGsxKtT06asVniUasIEHgI +MB0GA1UdDgQWBBRr4HG23dsao68r9r7obGxB70ptBzAOBgNVHQ8BAf8EBAMCB4Aw +DAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwSgYD +VR0gBEMwQTA1BgwrBgEEAbIxAQIBAwQwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9z +ZWN0aWdvLmNvbS9DUFMwCAYGZ4EMAQICMFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6 +Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb0VDQ09yZ2FuaXphdGlvblZhbGlkYXRp +b25TZWN1cmVTZXJ2ZXJDQS5jcmwwgYoGCCsGAQUFBwEBBH4wfDBVBggrBgEFBQcw +AoZJaHR0cDovL2NydC5zZWN0aWdvLmNvbS9TZWN0aWdvRUNDT3JnYW5pemF0aW9u +VmFsaWRhdGlvblNlY3VyZVNlcnZlckNBLmNydDAjBggrBgEFBQcwAYYXaHR0cDov +L29jc3Auc2VjdGlnby5jb20wMQYDVR0RBCowKIITKi50aGluZ3Nib2FyZC5jbG91 +ZIIRdGhpbmdzYm9hcmQuY2xvdWQwggGABgorBgEEAdZ5AgQCBIIBcASCAWwBagB2 +AEalVet1+pEgMLWiiWn0830RLEF0vv1JuIWr8vxw/m1HAAABfFDbhtMAAAQDAEcw +RQIhAKNykhkQTngK0yOcYHGHUQSy6JmJYl+5nc1qELirPHwAAiBgV2Db5ZFHNvzn +zp9Ob/OG0o36z6rcilbLI/daZwnyewB3AEHIyrHfIkZKEMahOglCh15OMYsbA+vr +S8do8JBilgb2AAABfFDbhqYAAAQDAEgwRgIhALFvTbapKhO7DPrF6KtE9sjFMMth +qjqaeaHYN6JGnUAIAiEApEW+rxlzxH1+qEwJrFyQLr5rSKTuEoSjv3hbrzb9GQ4A +dwApeb7wnjk5IfBWc59jpXflvld9nGAK+PlNXSZcJV3HhAAAAXxQ24ZnAAAEAwBI +MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL +1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ +GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR +13sr0/ED+2PUom2dnfg= +-----END CERTIFICATE----- \ No newline at end of file From 881cc886f9cd8a5ae99a7260d0cb6e521e247bff Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 21 Feb 2023 16:53:08 +0200 Subject: [PATCH 12/34] Add newline to end of files --- .../rule/engine/credentials/CertPemCredentialsTest.java | 2 +- .../rule-engine-components/src/test/resources/pem/tb-cloud.pem | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index e8126cade5..132b60554b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -55,4 +55,4 @@ public class CertPemCredentialsTest { File file = new File(classLoader.getResource(fileName).getFile()); return FileUtils.readFileToString(file, "UTF-8"); } -} \ No newline at end of file +} diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem index d75be0bf10..8aba29c995 100644 --- a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem @@ -29,4 +29,4 @@ MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL 1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR 13sr0/ED+2PUom2dnfg= ------END CERTIFICATE----- \ No newline at end of file +-----END CERTIFICATE----- From 1a6cd82461b306e1ffcb9c83669326f9ad41dc3a Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Feb 2023 13:29:15 +0200 Subject: [PATCH 13/34] License header fix --- .../credentials/CertPemCredentialsTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index 132b60554b..904f470001 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.rule.engine.credentials; import org.apache.commons.io.FileUtils; From d0fbecc4c5eba4e7fd4d1ffae364e96a05f4b693 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Feb 2023 16:10:53 +0200 Subject: [PATCH 14/34] Organize imports --- .../thingsboard/rule/engine/credentials/CertPemCredentials.java | 1 - 1 file changed, 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 480c4a0689..2593add406 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -60,7 +60,6 @@ import java.security.spec.RSAPrivateCrtKeySpec; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; From d93ba0e751ddade43299dc70c316af0ec6523f57 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Feb 2023 16:29:24 +0200 Subject: [PATCH 15/34] Code simplify - remove method --- .../rule/engine/credentials/CertPemCredentials.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 2593add406..0137887aaf 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -133,7 +133,7 @@ public class CertPemCredentials implements ClientCredentials { KeyStore clientKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); clientKeyStore.load(null, null); for (X509Certificate certHolder : certHolders) { - clientKeyStore.setCertificateEntry(createCertEntryAlias("cert-", certHolder), certHolder); + clientKeyStore.setCertificateEntry("cert-" + certHolder.getSubjectDN().getName(), certHolder); } clientKeyStore.setKeyEntry("private-key", privateKey, @@ -150,7 +150,7 @@ public class CertPemCredentials implements ClientCredentials { KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); for (X509Certificate caCertHolder : caCertHolders) { - caKeyStore.setCertificateEntry(createCertEntryAlias("caCert-cert-", caCertHolder), caCertHolder); + caKeyStore.setCertificateEntry("caCert-cert-" + caCertHolder.getSubjectDN().getName(), caCertHolder); } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); @@ -158,10 +158,6 @@ public class CertPemCredentials implements ClientCredentials { return trustManagerFactory; } - private String createCertEntryAlias(String prefix, X509Certificate certificate) { - return prefix + "-" + certificate.getSubjectDN().getName(); - } - List readCertFile(String fileContent) throws Exception { if (fileContent == null || fileContent.trim().isEmpty()) { return Collections.emptyList(); From 680f8662ed091de5f8e62cc6eed1772276a18dfd Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 23 Feb 2023 01:01:37 +0100 Subject: [PATCH 16/34] fixed processJsonTestGatewayRequestAttributesValuesFromTheServer --- .../AbstractMqttAttributesIntegrationTest.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index 2c0278f689..6d19e1261d 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.transport.mqtt.mqttv3.attributes; +import com.fasterxml.jackson.core.type.TypeReference; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; @@ -22,6 +23,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.squareup.wire.schema.internal.parser.ProtoFileElement; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DynamicProtoUtils; @@ -45,6 +47,7 @@ import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestClient; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -391,9 +394,19 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt 100); assertNotNull(device); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + + String attributeValuesUrl = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/attributes/CLIENT_SCOPE?keys=" + clientKeysStr; + + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> { + List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() {}); + return attributes.size() == 5; + }); + SingleEntityFilter dtf = new SingleEntityFilter(); dtf.setSingleEntity(device.getId()); - String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; List clientKeysList = List.of(clientKeysStr.split(",")); List sharedKeysList = List.of(sharedKeysStr.split(",")); From 4fd2a41087c68f397882e454098499a431155c76 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 24 Feb 2023 11:15:39 +0200 Subject: [PATCH 17/34] Remove EventDeduplicationExecutor --- .../utils/EventDeduplicationExecutor.java | 85 --------- .../util/EventDeduplicationExecutorTest.java | 173 ------------------ 2 files changed, 258 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java delete mode 100644 application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java diff --git a/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java b/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java deleted file mode 100644 index 7ce958ef2d..0000000000 --- a/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.utils; - -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.function.Consumer; - -/** - * This class deduplicate executions of the specified function. - * Useful in cluster mode, when you get event about partition change multiple times. - * Assuming that the function execution is expensive, we should execute it immediately when first time event occurs and - * later, once the processing of first event is done, process last pending task. - * - * @param

parameters of the function - */ -@Slf4j -public class EventDeduplicationExecutor

{ - private final String name; - private final ExecutorService executor; - private final Consumer

function; - private P pendingTask; - private boolean busy; - - public EventDeduplicationExecutor(String name, ExecutorService executor, Consumer

function) { - this.name = name; - this.executor = executor; - this.function = function; - } - - public void submit(P params) { - log.info("[{}] Going to submit: {}", name, params); - synchronized (EventDeduplicationExecutor.this) { - if (!busy) { - busy = true; - pendingTask = null; - try { - log.info("[{}] Submitting task: {}", name, params); - executor.submit(() -> { - try { - log.info("[{}] Executing task: {}", name, params); - function.accept(params); - } catch (Throwable e) { - log.warn("[{}] Failed to process task with parameters: {}", name, params, e); - throw e; - } finally { - unlockAndProcessIfAny(); - } - }); - } catch (Throwable e) { - log.warn("[{}] Failed to submit task with parameters: {}", name, params, e); - unlockAndProcessIfAny(); - throw e; - } - } else { - log.info("[{}] Task is already in progress. {} pending task: {}", name, pendingTask == null ? "adding" : "updating", params); - pendingTask = params; - } - } - } - - private void unlockAndProcessIfAny() { - synchronized (EventDeduplicationExecutor.this) { - busy = false; - if (pendingTask != null) { - submit(pendingTask); - } - } - } -} diff --git a/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java b/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java deleted file mode 100644 index ef75657ed4..0000000000 --- a/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.util; - -import com.google.common.util.concurrent.MoreExecutors; -import lombok.extern.slf4j.Slf4j; -import org.junit.After; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.utils.EventDeduplicationExecutor; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class EventDeduplicationExecutorTest { - - ThingsBoardThreadFactory threadFactory = ThingsBoardThreadFactory.forName(getClass().getSimpleName()); - ExecutorService executor; - - @After - public void tearDown() throws Exception { - if (executor != null) { - executor.shutdownNow(); - } - } - - @Test - public void testSimpleFlowSameThread() throws InterruptedException { - simpleFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testPeriodicFlowSameThread() throws InterruptedException { - periodicFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testExceptionFlowSameThread() throws InterruptedException { - exceptionFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testSimpleFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - simpleFlow(executor); - } - - @Test - public void testPeriodicFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - periodicFlow(executor); - } - - @Test - public void testExceptionFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - exceptionFlow(executor); - } - - @Test - public void testSimpleFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - simpleFlow(executor); - } - - @Test - public void testPeriodicFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - periodicFlow(executor); - } - - @Test - public void testExceptionFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - exceptionFlow(executor); - } - - private void simpleFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - executor.submit(params1); - executor.submit(params2); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params1); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - private void periodicFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - executor.submit(params1); - Thread.sleep(500); - executor.submit(params2); - Thread.sleep(500); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params1); - Mockito.verify(function).accept(params2); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - private void exceptionFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - Mockito.doThrow(new RuntimeException()).when(function).accept("params1"); - executor.submit(params1); - executor.submit(params2); - Thread.sleep(500); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params2); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - public static class StringConsumer implements Consumer { - @Override - public void accept(String s) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } - -} From 8a121b6b63aa3b295e348c51395429e048d1c457 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Feb 2023 20:23:26 +0100 Subject: [PATCH 18/34] tests: docker compose selenium - host.docker.internal:host-gateway --- msa/black-box-tests/src/test/resources/docker-selenium.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/resources/docker-selenium.yml b/msa/black-box-tests/src/test/resources/docker-selenium.yml index 38a3b74bfe..12b0489cd5 100644 --- a/msa/black-box-tests/src/test/resources/docker-selenium.yml +++ b/msa/black-box-tests/src/test/resources/docker-selenium.yml @@ -32,6 +32,4 @@ services: SE_SCREEN_DEPTH: 24 SE_SCREEN_DPI: 74 extra_hosts: - - "host.docker.internal:172.17.0.1" - - + - "host.docker.internal:host-gateway" From 60973c94221191d7d3786ea2ea0b03343b0a0a3b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Feb 2023 22:55:29 +0100 Subject: [PATCH 19/34] us tests: wait timeout increased up to 30 sec for slow or busy environments --- .../org/thingsboard/server/msa/ui/base/AbstractBasePage.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java index f8631bd198..cd4ca51698 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java @@ -32,9 +32,11 @@ import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; @Slf4j abstract public class AbstractBasePage { + public static final long WAIT_TIMEOUT = TimeUnit.SECONDS.toMillis(30); protected WebDriver driver; protected WebDriverWait wait; protected Actions actions; @@ -43,7 +45,7 @@ abstract public class AbstractBasePage { public AbstractBasePage(WebDriver driver) { this.driver = driver; - this.wait = new WebDriverWait(driver, Duration.ofMillis(8000)); + this.wait = new WebDriverWait(driver, Duration.ofMillis(WAIT_TIMEOUT)); this.actions = new Actions(driver); this.js = (JavascriptExecutor) driver; } From 0f324d113297ea7ffb56df3ab59172af423868ed Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 24 Feb 2023 11:02:29 +0100 Subject: [PATCH 20/34] HashPartitionService: set myPartitions atomically --- .../server/queue/discovery/HashPartitionService.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 31f81d5510..3fc1626f52 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -68,7 +68,7 @@ public class HashPartitionService implements PartitionService { private final TenantRoutingInfoService tenantRoutingInfoService; private final QueueRoutingInfoService queueRoutingInfoService; - private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); private final ConcurrentMap partitionTopicsMap = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizesMap = new ConcurrentHashMap<>(); @@ -217,17 +217,19 @@ public class HashPartitionService implements PartitionService { } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); - ConcurrentMap> oldPartitions = myPartitions; - myPartitions = new ConcurrentHashMap<>(); + final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i); if (currentService.equals(serviceInfo)) { - myPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); + newPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); } } }); + final ConcurrentMap> oldPartitions = myPartitions; + myPartitions = newPartitions; + oldPartitions.forEach((queueKey, partitions) -> { if (!myPartitions.containsKey(queueKey)) { log.info("[{}] NO MORE PARTITIONS FOR CURRENT KEY", queueKey); From 5edaec55a6a91df8070184d388c84cea1cd18be4 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 25 Feb 2023 15:15:03 +0100 Subject: [PATCH 21/34] ui tests: get reliable service url inside containers network --- .../test/java/org/thingsboard/server/msa/TestProperties.java | 5 ++++- msa/black-box-tests/src/test/resources/docker-selenium.yml | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java index 32d5b0f9d7..be9a5c422a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -16,6 +16,7 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.DockerClientFactory; import java.io.IOException; import java.io.InputStream; @@ -41,7 +42,9 @@ public class TestProperties { public static String getBaseUiUrl() { if (instance.isActive()) { - return "https://host.docker.internal"; + //return "https://host.docker.internal" // this alternative requires docker-selenium.yml extra_hosts: - "host.docker.internal:host-gateway" + //return "https://" + DockerClientFactory.instance().dockerHostIpAddress(); //this alternative will get Docker IP from testcontainers + return "https://haproxy"; //communicate inside current docker-compose network to the load balancer container } return getProperties().getProperty("tb.baseUiUrl"); } diff --git a/msa/black-box-tests/src/test/resources/docker-selenium.yml b/msa/black-box-tests/src/test/resources/docker-selenium.yml index 12b0489cd5..2a88264bc8 100644 --- a/msa/black-box-tests/src/test/resources/docker-selenium.yml +++ b/msa/black-box-tests/src/test/resources/docker-selenium.yml @@ -31,5 +31,6 @@ services: SE_SCREEN_HEIGHT: 1080 SE_SCREEN_DEPTH: 24 SE_SCREEN_DPI: 74 - extra_hosts: - - "host.docker.internal:host-gateway" +# Alternative way how to connect to the host address +# extra_hosts: +# - "host.docker.internal:host-gateway" From 028b1a023d3d4196ef07705b363b27bf4817b8fe Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 27 Feb 2023 14:21:39 +0100 Subject: [PATCH 22/34] fixed filtering by entity name --- .../query/DefaultEntityQueryRepository.java | 10 + .../dao/service/BaseEntityServiceTest.java | 360 +++++++++++++++++- 2 files changed, 366 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 681efe4829..821dbcc011 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -805,6 +805,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private String entityNameQuery(QueryContext ctx, EntityNameFilter filter) { ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter()); + + if (filter.getEntityNameFilter().startsWith("%") || filter.getEntityNameFilter().endsWith("%")) { + return "lower(e.search_text) like lower(:entity_filter_name_filter)"; + } + return "lower(e.search_text) like lower(concat(:entity_filter_name_filter, '%%'))"; } @@ -833,6 +838,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } ctx.addStringParameter("entity_filter_type_query_type", type); ctx.addStringParameter("entity_filter_type_query_name", name); + + if (name.startsWith("%") || name.endsWith("%")) { + return "e.type = :entity_filter_type_query_type and lower(e.search_text) like lower(:entity_filter_type_query_name)"; + } + return "e.type = :entity_filter_type_query_type and lower(e.search_text) like lower(concat(:entity_filter_type_query_name, '%%'))"; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index e2003ed7b2..7c80bef095 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -26,7 +26,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -50,6 +49,7 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.AssetTypeFilter; import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; @@ -62,6 +62,7 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; @@ -106,9 +107,6 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { private TenantId tenantId; - @Autowired - private JdbcTemplate template; - @Autowired private RelationRepository relationRepository; @@ -986,6 +984,360 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { assertEquals(devices.size(), result.getTotalElements()); } + @Test + public void testFindEntityDataByQuery_filter_entity_name_starts_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("Device"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%Device"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_entity_name_ends_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_entity_name_contains() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device test" + i); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_starts_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("Device"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%Device"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_ends_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_contains() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device test" + i); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_starts_with() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset " + i + " test"); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("Asset"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("Asset%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%Asset%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%Asset"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_ends_with() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset " + i + " test"); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_contains() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset test" + i); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + private PageData searchEntities(EntityDataQuery query) { return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); } From f823021ab8f4f075473cccb80bff5a70fd6ee93e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 13:39:55 +0100 Subject: [PATCH 23/34] msa black box tests: cassandra limited to MAX_HEAP_SIZE: 1024M --- .../server/msa/ContainerTestSuite.java | 1 + .../server/msa/ThingsBoardDbInstaller.java | 3 +++ .../docker-compose.hybrid-test-extras.yml | 23 +++++++++++++++++++ .../docker-compose.postgres-test-extras.yml | 19 +++++++++++++++ 4 files changed, 46 insertions(+) create mode 100644 msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml create mode 100644 msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 95e2877904..ffbe14ed52 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -106,6 +106,7 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), + new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 4b07a72772..e6c9856094 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -82,6 +82,9 @@ public class ThingsBoardDbInstaller { )); if (IS_HYBRID_MODE) { composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); + composeFiles.add(new File("src/test/resources/docker-compose.hybrid-test-extras.yml")); + } else { + composeFiles.add(new File("src/test/resources/docker-compose.postgres-test-extras.yml")); } String identifier = Base58.randomString(6).toLowerCase(); diff --git a/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml new file mode 100644 index 0000000000..3b5e61b47f --- /dev/null +++ b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + cassandra: + environment: + HEAP_NEWSIZE: 128M + MAX_HEAP_SIZE: 1024M diff --git a/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml new file mode 100644 index 0000000000..6576aa47b7 --- /dev/null +++ b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml @@ -0,0 +1,19 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +# Placeholder From 6fc6ef7b0627d321fc3da5750e66368e36892599 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 17:42:30 +0100 Subject: [PATCH 24/34] msa black-box-tests replaced docker compose version '2.2' -> '3.0' --- .../src/test/resources/docker-compose.hybrid-test-extras.yml | 2 +- .../src/test/resources/docker-compose.postgres-test-extras.yml | 2 +- .../src/test/resources/docker-compose.rabbitmq-server.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml index 3b5e61b47f..5df6d2582d 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml @@ -14,7 +14,7 @@ # limitations under the License. # -version: '2.2' +version: '3.0' services: cassandra: diff --git a/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml index 6576aa47b7..90d7a351cf 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml @@ -14,6 +14,6 @@ # limitations under the License. # -version: '2.2' +version: '3.0' # Placeholder diff --git a/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml b/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml index 414aef5568..f1c2f0b41c 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml @@ -14,7 +14,7 @@ # limitations under the License. # -version: '2.2' +version: '3.0' services: rabbitmq: From 7e1ba7dec5514e0c1cbac321760d53c9fb4c5357 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Feb 2023 15:55:27 +0100 Subject: [PATCH 25/34] Test mail service replaced with MockBean to fix the app lifecycle --- .../server/controller/AbstractWebTest.java | 44 ++++++++++++-- .../controller/BaseAdminControllerTest.java | 22 ++----- .../controller/BaseUserControllerTest.java | 17 +++--- .../server/service/mail/TestMailService.java | 58 ------------------- 4 files changed, 55 insertions(+), 86 deletions(-) delete mode 100644 application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 24ce820760..ab92766a27 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -36,7 +36,10 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -52,6 +55,9 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; @@ -69,6 +75,7 @@ import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -83,7 +90,6 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.config.ThingsboardSecurityConfiguration; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.tenant.TenantProfileService; -import org.thingsboard.server.service.mail.TestMailService; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest; import org.thingsboard.server.service.security.auth.rest.LoginRequest; @@ -98,6 +104,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; @@ -144,6 +151,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected MockMvc mockMvc; + protected String currentActivateToken; + protected String currentResetPasswordToken; + protected String token; protected String refreshToken; protected String username; @@ -168,6 +178,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { @Autowired private TenantProfileService tenantProfileService; + @SpyBean + protected MailService mailService; + @Rule public TestRule watcher = new TestWatcher() { protected void starting(Description description) { @@ -200,6 +213,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { public void setupWebTest() throws Exception { log.info("Executing web test setup"); + setupMailServiceMock(); + if (this.mockMvc == null) { this.mockMvc = webAppContextSetup(webApplicationContext) .apply(springSecurity()).build(); @@ -241,6 +256,27 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { log.info("Executed web test setup"); } + private void setupMailServiceMock() throws ThingsboardException { + Mockito.doNothing().when(mailService).sendAccountActivatedEmail(anyString(), anyString()); + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + String activationLink = (String) args[0]; + currentActivateToken = activationLink.split("=")[1]; + return null; + } + }).when(mailService).sendActivationEmail(anyString(), anyString()); + + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + String passwordResetLink = (String) args[0]; + currentResetPasswordToken = passwordResetLink.split("=")[1]; + return null; + } + }).when(mailService).sendResetPasswordEmailAsync(anyString(), anyString()); + } + @After public void teardownWebTest() throws Exception { log.info("Executing web test teardown"); @@ -368,11 +404,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } private JsonNode getActivateRequest(String password) throws Exception { - doGet("/api/noauth/activate?activateToken={activateToken}", TestMailService.currentActivateToken) + doGet("/api/noauth/activate?activateToken={activateToken}", this.currentActivateToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + TestMailService.currentActivateToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + this.currentActivateToken)); return new ObjectMapper().createObjectNode() - .put("activateToken", TestMailService.currentActivateToken) + .put("activateToken", this.currentActivateToken) .put("password", password); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java index 021171f1af..e6c27aa4a8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java @@ -21,12 +21,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Test; import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.security.model.JwtSettings; -import org.thingsboard.server.service.mail.DefaultMailService; import java.nio.charset.StandardCharsets; import java.util.Base64; @@ -35,6 +32,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -43,12 +42,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. public abstract class BaseAdminControllerTest extends AbstractControllerTest { final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "thingsboardDefaultSigningKey"); - @Autowired - MailService mailService; - - @Autowired - DefaultMailService defaultMailService; - @Test public void testFindAdminSettingsByKey() throws Exception { loginSysAdmin(); @@ -118,10 +111,12 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { @Test public void testSendTestMail() throws Exception { + Mockito.doNothing().when(mailService).sendTestMail(any(), anyString()); loginSysAdmin(); AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); doPost("/api/admin/settings/testMail", adminSettings) .andExpect(status().isOk()); + Mockito.verify(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); } @Test @@ -137,15 +132,8 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { adminSettings.setJsonValue(objectNode); - Mockito.doAnswer((invocations) -> { - var jsonConfig = (JsonNode) invocations.getArgument(0); - var email = (String) invocations.getArgument(1); - - defaultMailService.sendTestMail(jsonConfig, email); - return null; - }).when(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); doPost("/api/admin/settings/testMail", adminSettings).andExpect(status().is5xxServerError()); - Mockito.doNothing().when(mailService).sendTestMail(Mockito.any(), Mockito.any()); + Mockito.verify(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); } void resetJwtSettingsToDefault() throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index f289c8066f..7149f52630 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.user.UserDao; -import org.thingsboard.server.service.mail.TestMailService; import java.util.ArrayList; import java.util.Collections; @@ -54,6 +53,8 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -106,12 +107,12 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); resetTokens(); - doGet("/api/noauth/activate?activateToken={activateToken}", TestMailService.currentActivateToken) + doGet("/api/noauth/activate?activateToken={activateToken}", this.currentActivateToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + TestMailService.currentActivateToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + this.currentActivateToken)); JsonNode activateRequest = new ObjectMapper().createObjectNode() - .put("activateToken", TestMailService.currentActivateToken) + .put("activateToken", this.currentActivateToken) .put("password", "testPassword"); JsonNode tokenInfo = readResponse(doPost("/api/noauth/activate", activateRequest).andExpect(status().isOk()), JsonNode.class); @@ -208,17 +209,19 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { doPost("/api/noauth/resetPasswordByEmail", resetPasswordByEmailRequest) .andExpect(status().isOk()); Thread.sleep(1000); - doGet("/api/noauth/resetPassword?resetToken={resetToken}", TestMailService.currentResetPasswordToken) + doGet("/api/noauth/resetPassword?resetToken={resetToken}", this.currentResetPasswordToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/resetPassword?resetToken=" + TestMailService.currentResetPasswordToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/resetPassword?resetToken=" + this.currentResetPasswordToken)); JsonNode resetPasswordRequest = new ObjectMapper().createObjectNode() - .put("resetToken", TestMailService.currentResetPasswordToken) + .put("resetToken", this.currentResetPasswordToken) .put("password", "testPassword2"); + Mockito.doNothing().when(mailService).sendPasswordWasResetEmail(anyString(), anyString()); JsonNode tokenInfo = readResponse( doPost("/api/noauth/resetPassword", resetPasswordRequest) .andExpect(status().isOk()), JsonNode.class); + Mockito.verify(mailService).sendPasswordWasResetEmail(anyString(), anyString()); validateAndSetJwtToken(tokenInfo, email); doGet("/api/auth/user") diff --git a/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java b/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java deleted file mode 100644 index 7c4c45c951..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.mail; - -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Profile; -import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.server.common.data.exception.ThingsboardException; - -@Profile("test") -@Configuration -public class TestMailService { - - public static String currentActivateToken; - public static String currentResetPasswordToken; - - @Bean - @Primary - public MailService mailService() throws ThingsboardException { - MailService mailService = Mockito.mock(MailService.class); - Mockito.doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String activationLink = (String) args[0]; - currentActivateToken = activationLink.split("=")[1]; - return null; - } - }).when(mailService).sendActivationEmail(Mockito.anyString(), Mockito.anyString()); - Mockito.doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String passwordResetLink = (String) args[0]; - currentResetPasswordToken = passwordResetLink.split("=")[1]; - return null; - } - }).when(mailService).sendResetPasswordEmailAsync(Mockito.anyString(), Mockito.anyString()); - return mailService; - } - -} From a6a4a67a3d3b644803ed23ecaf9c6b6d9b7d2cb0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 17 Feb 2023 17:50:50 +0100 Subject: [PATCH 26/34] handle timeouts on transport tests to provide clear explanation what is the problem. timeout increased for run test stable on slow (busy) environments --- .../AbstractTransportIntegrationTest.java | 2 +- ...AbstractMqttAttributesIntegrationTest.java | 49 ++++++++++++------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java index cf4de21cc2..89a4b707bf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java @@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; @Slf4j public abstract class AbstractTransportIntegrationTest extends AbstractControllerTest { - protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 10; + protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 30; protected static final String MQTT_URL = "tcp://localhost:1883"; protected static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/"; diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index 6d19e1261d..7feafead95 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -128,14 +128,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateUpdateAttributesJsonResponse(onUpdateCallback, SHARED_ATTRIBUTES_PAYLOAD); MqttTestCallback onDeleteCallback = new MqttTestCallback(); client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateUpdateAttributesJsonResponse(onDeleteCallback, SHARED_ATTRIBUTES_DELETED_RESPONSE); client.disconnect(); } @@ -148,13 +150,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateUpdateAttributesProtoResponse(onUpdateCallback); MqttTestCallback onDeleteCallback = new MqttTestCallback(); client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateDeleteAttributesProtoResponse(onDeleteCallback); client.disconnect(); } @@ -165,7 +169,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateUpdateAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); List tsKvProtoList = getTsKvProtoList("shared"); attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); @@ -181,7 +185,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateDeleteAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); @@ -212,7 +216,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(GATEWAY_ATTRIBUTES_TOPIC, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateJsonGatewayUpdateAttributesResponse(onUpdateCallback, deviceName, SHARED_ATTRIBUTES_PAYLOAD); @@ -220,7 +225,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateJsonGatewayUpdateAttributesResponse(onDeleteCallback, deviceName, SHARED_ATTRIBUTES_DELETED_RESPONSE); client.disconnect(); @@ -249,7 +255,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName, String expectResultData) { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); assertEquals(JacksonUtil.toJsonNode(getGatewayAttributesResponseJson(deviceName, expectResultData)), JacksonUtil.fromBytes(callback.getPayloadBytes())); } @@ -263,8 +269,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); List tsKvProtoList = getTsKvProtoList("shared"); @@ -288,8 +295,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoGatewayDeleteAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); TransportProtos.AttributeUpdateNotificationMsg attributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); @@ -551,13 +559,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonResponse(MqttTestCallback callback, String expectedResponse) throws InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(callback.getPayloadBytes())); } protected void validateProtoResponse(MqttTestCallback callback, TransportProtos.GetAttributeResponseMsg expectedResponse) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); assertEquals(expectedResponse.getRequestId(), actualAttributesResponse.getRequestId()); @@ -580,14 +590,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonResponseGateway(MqttTestCallback callback, String deviceName, String expectedValues) throws InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); String expectedRequestPayload = "{\"id\":1,\"device\":\"" + deviceName + "\",\"values\":" + expectedValues + "}"; assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.fromBytes(callback.getPayloadBytes())); } protected void validateProtoClientResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, true); TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); @@ -603,7 +615,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoSharedResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, false); TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); From c38dd1270381816f69a45b583a94c94ee2ac1bc4 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:39:08 +0100 Subject: [PATCH 27/34] test dependency upgrade: surefire.version 3.0.0-M9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 61053bebc7..ae5dba306e 100755 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ 2.0.51.Final 1.7.0 4.8.0 - 3.0.0-M6 + 3.0.0-M9 3.0.2 3.0.4 1.6.3 From af62bc1d98361abda60f86e6688e0e3ed380f563 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:38:02 +0100 Subject: [PATCH 28/34] Shutdown jsExecutor after Nashorn destroy --- .../org/thingsboard/script/api/js/NashornJsInvokeService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java index c995474452..a93bb018d8 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java @@ -127,6 +127,9 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { if (monitorExecutorService != null) { monitorExecutorService.shutdownNow(); } + if (jsExecutor != null) { + jsExecutor.shutdownNow(); + } } @Override From bf5f11f7c7d1e70b5470e41f8f4d43fcdea95959 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 21:36:24 +0100 Subject: [PATCH 29/34] tests: surefire plugin: -Xss384k -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=20 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae5dba306e..7d6b59f10b 100755 --- a/pom.xml +++ b/pom.xml @@ -659,7 +659,7 @@ ${surefire.version} - --illegal-access=permit + --illegal-access=permit -Xss384k -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=20 From 0448c9aa03135ddd92720b31c272eef7f0f847d7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 18 Feb 2023 00:16:12 +0100 Subject: [PATCH 30/34] test memory optimization (disable integration rpc, coap, hikari.maximumPoolSize=16, etc). --- .../server/controller/AbstractWebTest.java | 19 ++++++++++++++++--- .../BaseWidgetsBundleControllerTest.java | 2 +- .../resources/application-test.properties | 8 ++++++++ .../src/test/resources/logback-test.xml | 1 + dao/src/test/resources/nosql-test.properties | 2 +- dao/src/test/resources/sql-test.properties | 2 +- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 24ce820760..8e0804e3df 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -29,8 +29,10 @@ import lombok.extern.slf4j.Slf4j; import org.hamcrest.Matcher; import org.hibernate.exception.ConstraintViolationException; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; @@ -196,9 +198,20 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { this.mappingJackson2HttpMessageConverter); } + @BeforeClass + public static void beforeWebTestClass() throws Exception { + + } + + @AfterClass + public static void afterWebTestClass() throws Exception { + Mockito.clearAllCaches(); + } + + @Before public void setupWebTest() throws Exception { - log.info("Executing web test setup"); + log.debug("Executing web test setup"); if (this.mockMvc == null) { this.mockMvc = webAppContextSetup(webApplicationContext) @@ -238,12 +251,12 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { resetTokens(); - log.info("Executed web test setup"); + log.debug("Executed web test setup"); } @After public void teardownWebTest() throws Exception { - log.info("Executing web test teardown"); + log.debug("Executing web test teardown"); loginSysAdmin(); doDelete("/api/tenant/" + tenantId.getId().toString()) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java index 56df8513fe..828d279e93 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java @@ -192,7 +192,7 @@ public abstract class BaseWidgetsBundleControllerTest extends AbstractController WidgetsBundle savedWidgetsBundle = doPost("/api/widgetsBundle", widgetsBundle, WidgetsBundle.class); savedWidgetsBundle.setAlias("new_alias"); - Mockito.reset(tbClusterService); + Mockito.clearInvocations(tbClusterService); doPost("/api/widgetsBundle", savedWidgetsBundle) .andExpect(status().isBadRequest()) diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index e4a36c151d..330ae69787 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -20,6 +20,8 @@ transport.mqtt.enabled=false transport.coap.enabled=false transport.lwm2m.enabled=false transport.snmp.enabled=false +coap.enabled=false +integrations.rpc.enabled=false # Low latency settings to perform tests as fast as possible sql.attributes.batch_max_delay=5 @@ -63,3 +65,9 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 +#spring.jpa.properties.hibernate.generate_statistics=true +#spring.jpa.properties.hibernate.jmx.enabled=true +#spring.jpa.properties.hibernate.jmx.usePlatformServer=true +#spring.jpa.properties.hibernate.cache.use_query_cache=false +#spring.jpa.properties.hibernate.query.plan_cache_max_size=64 +#spring.jpa.properties.hibernate.query.plan_parameter_metadata_max_size=32 diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 3762c8aa7c..c79a7d3905 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -15,6 +15,7 @@ + diff --git a/dao/src/test/resources/nosql-test.properties b/dao/src/test/resources/nosql-test.properties index 1c6d2442cf..d63a9f7f82 100644 --- a/dao/src/test/resources/nosql-test.properties +++ b/dao/src/test/resources/nosql-test.properties @@ -15,7 +15,7 @@ spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver -spring.datasource.hikari.maximumPoolSize = 50 +spring.datasource.hikari.maximumPoolSize=16 queue.rule-engine.queues[0].name=Main queue.rule-engine.queues[0].topic=tb_rule_engine.main diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index b2ded716c4..2fed33b43a 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,7 +16,7 @@ spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver -spring.datasource.hikari.maximumPoolSize = 50 +spring.datasource.hikari.maximumPoolSize=16 service.type=monolith From 368455119b03e7c5a829a95eed30cd83ed01bef2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 21:58:12 +0100 Subject: [PATCH 31/34] Shutdown callback executor for RemoteJsInvokeService. Call shutdown timeoutExecutorService from AbstractScriptInvokeService --- .../server/service/script/RemoteJsInvokeService.java | 5 ++++- .../thingsboard/script/api/js/NashornJsInvokeService.java | 2 ++ .../script/api/tbel/DefaultTbelInvokeService.java | 5 ++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 82c806ad4f..8fff7fc0fe 100644 --- a/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -109,17 +109,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final Lock scriptsLock = new ReentrantLock(); @PostConstruct + @Override public void init() { super.init(); requestTemplate.init(); } @PreDestroy - public void destroy() { + @Override + public void stop() { super.stop(); if (requestTemplate != null) { requestTemplate.stop(); } + callbackExecutor.shutdownNow(); } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java index c995474452..b1f1c0d2b1 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java @@ -104,6 +104,7 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { } @PostConstruct + @Override public void init() { super.init(); jsExecutor = MoreExecutors.listeningDecorator(Executors.newWorkStealingPool(jsExecutorThreadPoolSize)); @@ -122,6 +123,7 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { } @PreDestroy + @Override public void stop() { super.stop(); if (monitorExecutorService != null) { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 7d2043557e..f98e33b615 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -119,6 +119,7 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem @SneakyThrows @PostConstruct + @Override public void init() { super.init(); OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); @@ -142,7 +143,9 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem } @PreDestroy - public void destroy() { + @Override + public void stop() { + super.stop(); if (executor != null) { executor.shutdownNow(); } From 12c0e23ed3514564e0fe547bc506574eab2bbc14 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:48:42 +0100 Subject: [PATCH 32/34] disabled service.integrations and js.evaluator for the most of cases --- .../server/service/script/NashornJsInvokeServiceTest.java | 1 + .../transport/coap/AbstractCoapIntegrationTest.java | 2 ++ .../transport/mqtt/AbstractMqttIntegrationTest.java | 2 +- .../src/test/resources/application-test.properties | 8 ++------ 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java index 08cd031a52..4373a4c108 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java @@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest @TestPropertySource(properties = { + "js.evaluator=local", "js.max_script_body_size=50", "js.max_total_args_size=50", "js.max_result_size=50", diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index d90af7b6ba..1266ef2c29 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -45,6 +45,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @TestPropertySource(properties = { + "coap.enabled=true", + "service.integrations.supported=ALL", "transport.coap.enabled=true", }) @Slf4j diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java index 0f1815b46b..6cb6e49e79 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java @@ -46,8 +46,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @TestPropertySource(properties = { + "service.integrations.supported=ALL", "transport.mqtt.enabled=true", - "js.evaluator=mock", }) @Slf4j public abstract class AbstractMqttIntegrationTest extends AbstractTransportIntegrationTest { diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 330ae69787..ad86ff736b 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -1,3 +1,4 @@ +js.evaluator=mock transport.lwm2m.server.security.credentials.enabled=true transport.lwm2m.server.security.credentials.type=KEYSTORE transport.lwm2m.server.security.credentials.keystore.store_file=lwm2m/credentials/lwm2mserver.jks @@ -22,6 +23,7 @@ transport.lwm2m.enabled=false transport.snmp.enabled=false coap.enabled=false integrations.rpc.enabled=false +service.integrations.supported=NONE # Low latency settings to perform tests as fast as possible sql.attributes.batch_max_delay=5 @@ -65,9 +67,3 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 -#spring.jpa.properties.hibernate.generate_statistics=true -#spring.jpa.properties.hibernate.jmx.enabled=true -#spring.jpa.properties.hibernate.jmx.usePlatformServer=true -#spring.jpa.properties.hibernate.cache.use_query_cache=false -#spring.jpa.properties.hibernate.query.plan_cache_max_size=64 -#spring.jpa.properties.hibernate.query.plan_parameter_metadata_max_size=32 From 24c930c699f65518f76947f4e2a36326a762cd3d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:49:08 +0100 Subject: [PATCH 33/34] spring.test.context.cache.maxSize=1 --- application/src/test/resources/application-test.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index ad86ff736b..a4bedd9527 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -67,3 +67,4 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 +spring.test.context.cache.maxSize=1 \ No newline at end of file From 945d1955e2de09d61d1c4684d94b590c503268f0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 00:26:06 +0100 Subject: [PATCH 34/34] msa modules reordered to tolerate parallel build with 3 parallel threads --- msa/pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/msa/pom.xml b/msa/pom.xml index 515557b018..ad40e574ec 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -39,13 +39,14 @@ + tb + web-ui vc-executor vc-executor-docker - js-executor - web-ui tb-node transport + js-executor