diff --git a/application/pom.xml b/application/pom.xml index 0d9982f281..b653465050 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -144,21 +144,6 @@ org.eclipse.paho org.eclipse.paho.mqttv5.client - - org.cassandraunit - cassandra-unit - - - org.slf4j - slf4j-log4j12 - - - org.hibernate - hibernate-validator - - - test - org.thingsboard ui-ngx @@ -329,6 +314,11 @@ spring-test-dbunit test + + org.testcontainers + cassandra + test + org.testcontainers postgresql diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 47355d6c44..9a6ffb0e32 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.config; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.HttpStatus; @@ -40,11 +42,15 @@ import java.util.Map; @Configuration @TbCoreComponent @EnableWebSocket +@RequiredArgsConstructor +@Slf4j public class WebSocketConfiguration implements WebSocketConfigurer { public static final String WS_PLUGIN_PREFIX = "/api/ws/plugins/"; private static final String WS_PLUGIN_MAPPING = WS_PLUGIN_PREFIX + "**"; + private final WebSocketHandler wsHandler; + @Bean public ServletServerContainerFactoryBean createWebSocketContainer() { ServletServerContainerFactoryBean container = new ServletServerContainerFactoryBean(); @@ -55,7 +61,11 @@ public class WebSocketConfiguration implements WebSocketConfigurer { @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(wsHandler(), WS_PLUGIN_MAPPING).setAllowedOriginPatterns("*") + if (!(wsHandler instanceof TbWebSocketHandler)) { + log.error("TbWebSocketHandler expected but [{}] provided", wsHandler); + throw new RuntimeException("TbWebSocketHandler expected but " + wsHandler + " provided"); + } + registry.addHandler(wsHandler, WS_PLUGIN_MAPPING).setAllowedOriginPatterns("*") .addInterceptors(new HttpSessionHandshakeInterceptor(), new HandshakeInterceptor() { @Override @@ -82,11 +92,6 @@ public class WebSocketConfiguration implements WebSocketConfigurer { }); } - @Bean - public WebSocketHandler wsHandler() { - return new TbWebSocketHandler(); - } - protected SecurityUser getCurrentUser() throws ThingsboardException { Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); if (authentication != null && authentication.getPrincipal() instanceof SecurityUser) { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index c73ac1ae66..113c292380 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -20,6 +20,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -41,11 +42,13 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent; import org.thingsboard.server.common.data.security.event.UserSessionInvalidationEvent; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; +import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; @@ -62,6 +65,8 @@ import org.thingsboard.server.service.security.system.SystemSecurityService; import javax.servlet.http.HttpServletRequest; import java.net.URI; import java.net.URISyntaxException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; @RestController @TbCoreComponent @@ -69,6 +74,10 @@ import java.net.URISyntaxException; @Slf4j @RequiredArgsConstructor public class AuthController extends BaseController { + + @Value("${server.rest.rate_limits.reset_password_per_user:5:3600}") + private String defaultLimitsConfiguration; + private final ConcurrentMap resetPasswordRateLimits = new ConcurrentHashMap<>(); private final BCryptPasswordEncoder passwordEncoder; private final JwtTokenFactory tokenFactory; private final MailService mailService; @@ -211,7 +220,12 @@ public class AuthController extends BaseController { HttpStatus responseStatus; String resetURI = "/login/resetPassword"; UserCredentials userCredentials = userService.findUserCredentialsByResetToken(TenantId.SYS_TENANT_ID, resetToken); + if (userCredentials != null) { + TbRateLimits tbRateLimits = getTbRateLimits(userCredentials.getUserId()); + if (!tbRateLimits.tryConsume()) { + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); + } try { URI location = new URI(resetURI + "?resetToken=" + resetToken); headers.setLocation(location); @@ -323,4 +337,9 @@ public class AuthController extends BaseController { throw handleException(e); } } + + private TbRateLimits getTbRateLimits(UserId userId) { + return resetPasswordRateLimits.computeIfAbsent(userId, + key -> new TbRateLimits(defaultLimitsConfiguration, true)); + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index c689ae3593..f43607b24c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -67,8 +67,8 @@ import static org.thingsboard.server.service.telemetry.DefaultTelemetryWebSocket @Slf4j public class TbWebSocketHandler extends TextWebSocketHandler implements TelemetryWebSocketMsgEndpoint { - private static final ConcurrentMap internalSessionMap = new ConcurrentHashMap<>(); - private static final ConcurrentMap externalSessionMap = new ConcurrentHashMap<>(); + private final ConcurrentMap internalSessionMap = new ConcurrentHashMap<>(); + private final ConcurrentMap externalSessionMap = new ConcurrentHashMap<>(); @Autowired @@ -82,13 +82,13 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr @Value("${server.ws.ping_timeout:30000}") private long pingTimeout; - private ConcurrentMap blacklistedSessions = new ConcurrentHashMap<>(); - private ConcurrentMap perSessionUpdateLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap blacklistedSessions = new ConcurrentHashMap<>(); + private final ConcurrentMap perSessionUpdateLimits = new ConcurrentHashMap<>(); - private ConcurrentMap> tenantSessionsMap = new ConcurrentHashMap<>(); - private ConcurrentMap> customerSessionsMap = new ConcurrentHashMap<>(); - private ConcurrentMap> regularUserSessionsMap = new ConcurrentHashMap<>(); - private ConcurrentMap> publicUserSessionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantSessionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> customerSessionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> regularUserSessionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> publicUserSessionsMap = new ConcurrentHashMap<>(); @Override public void handleTextMessage(WebSocketSession session, TextMessage message) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index a2a0d0fd8e..ffde6192ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.telemetry; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -27,6 +26,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.socket.CloseStatus; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DataConstants; @@ -95,6 +95,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -112,7 +114,6 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private static final Aggregation DEFAULT_AGGREGATION = Aggregation.NONE; private static final int UNKNOWN_SUBSCRIPTION_ID = 0; private static final String PROCESSING_MSG = "[{}] Processing: {}"; - private static final ObjectMapper jsonMapper = new ObjectMapper(); private static final String FAILED_TO_FETCH_DATA = "Failed to fetch data!"; private static final String FAILED_TO_FETCH_ATTRIBUTES = "Failed to fetch attributes!"; private static final String SESSION_META_DATA_NOT_FOUND = "Session meta-data not found!"; @@ -147,10 +148,10 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Value("${server.ws.ping_timeout:30000}") private long pingTimeout; - private ConcurrentMap> tenantSubscriptionsMap = new ConcurrentHashMap<>(); - private ConcurrentMap> customerSubscriptionsMap = new ConcurrentHashMap<>(); - private ConcurrentMap> regularUserSubscriptionsMap = new ConcurrentHashMap<>(); - private ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantSubscriptionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> customerSubscriptionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> regularUserSubscriptionsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> publicUserSubscriptionsMap = new ConcurrentHashMap<>(); private ExecutorService executor; private String serviceId; @@ -204,7 +205,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi } try { - TelemetryPluginCmdsWrapper cmdsWrapper = jsonMapper.readValue(msg, TelemetryPluginCmdsWrapper.class); + TelemetryPluginCmdsWrapper cmdsWrapper = JacksonUtil.OBJECT_MAPPER.readValue(msg, TelemetryPluginCmdsWrapper.class); if (cmdsWrapper != null) { if (cmdsWrapper.getAttrSubCmds() != null) { cmdsWrapper.getAttrSubCmds().forEach(cmd -> { @@ -450,7 +451,6 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { List attributesData = data.stream().map(d -> new BasicTsKvEntry(d.getLastUpdateTs(), d)).collect(Collectors.toList()); - sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), attributesData)); Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, 0L)); @@ -458,6 +458,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.ANY_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + Lock subLock = new ReentrantLock(); TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -467,9 +468,24 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .allKeys(false) .keyStates(subState) .scope(scope) - .updateConsumer(DefaultTelemetryWebSocketService.this::sendWsMsg) + .updateConsumer((sessionId, update) -> { + subLock.lock(); + try { + sendWsMsg(sessionId, update); + } finally { + subLock.unlock(); + } + }) .build(); - oldSubService.addSubscription(sub); + + subLock.lock(); + try{ + oldSubService.addSubscription(sub); + sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), attributesData)); + } finally { + subLock.unlock(); + } + } @Override @@ -550,13 +566,13 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @Override public void onSuccess(List data) { List attributesData = data.stream().map(d -> new BasicTsKvEntry(d.getLastUpdateTs(), d)).collect(Collectors.toList()); - sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), attributesData)); Map subState = new HashMap<>(attributesData.size()); attributesData.forEach(v -> subState.put(v.getKey(), v.getTs())); TbAttributeSubscriptionScope scope = StringUtils.isEmpty(cmd.getScope()) ? TbAttributeSubscriptionScope.ANY_SCOPE : TbAttributeSubscriptionScope.valueOf(cmd.getScope()); + Lock subLock = new ReentrantLock(); TbAttributeSubscription sub = TbAttributeSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) @@ -565,9 +581,24 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi .entityId(entityId) .allKeys(true) .keyStates(subState) - .updateConsumer(DefaultTelemetryWebSocketService.this::sendWsMsg) - .scope(scope).build(); - oldSubService.addSubscription(sub); + .updateConsumer((sessionId, update) -> { + subLock.lock(); + try { + sendWsMsg(sessionId, update); + } finally { + subLock.unlock(); + } + }) + .scope(scope) + .build(); + + subLock.lock(); + try { + oldSubService.addSubscription(sub); + sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), attributesData)); + } finally { + subLock.unlock(); + } } @Override @@ -636,20 +667,34 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi FutureCallback> callback = new FutureCallback>() { @Override public void onSuccess(List data) { - sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(data.size()); data.forEach(v -> subState.put(v.getKey(), v.getTs())); + Lock subLock = new ReentrantLock(); TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) .subscriptionId(cmd.getCmdId()) .tenantId(sessionRef.getSecurityCtx().getTenantId()) .entityId(entityId) - .updateConsumer(DefaultTelemetryWebSocketService.this::sendWsMsg) + .updateConsumer((sessionId, update) -> { + subLock.lock(); + try { + sendWsMsg(sessionId, update); + } finally { + subLock.unlock(); + } + }) .allKeys(true) .keyStates(subState).build(); - oldSubService.addSubscription(sub); + + subLock.lock(); + try { + oldSubService.addSubscription(sub); + sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), data)); + } finally { + subLock.unlock(); + } } @Override @@ -673,21 +718,35 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi return new FutureCallback<>() { @Override public void onSuccess(List data) { - sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), data)); Map subState = new HashMap<>(keys.size()); keys.forEach(key -> subState.put(key, startTs)); data.forEach(v -> subState.put(v.getKey(), v.getTs())); + Lock subLock = new ReentrantLock(); TbTimeseriesSubscription sub = TbTimeseriesSubscription.builder() .serviceId(serviceId) .sessionId(sessionId) .subscriptionId(cmd.getCmdId()) .tenantId(sessionRef.getSecurityCtx().getTenantId()) .entityId(entityId) - .updateConsumer(DefaultTelemetryWebSocketService.this::sendWsMsg) + .updateConsumer((sessionId, update) -> { + subLock.lock(); + try { + sendWsMsg(sessionId, update); + } finally { + subLock.unlock(); + } + }) .allKeys(false) .keyStates(subState).build(); - oldSubService.addSubscription(sub); + + subLock.lock(); + try{ + oldSubService.addSubscription(sub); + sendWsMsg(sessionRef, new TelemetrySubscriptionUpdate(cmd.getCmdId(), data)); + } finally { + subLock.unlock(); + } } @Override @@ -793,7 +852,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi private void sendWsMsg(TelemetryWebSocketSessionRef sessionRef, int cmdId, Object update) { try { - String msg = jsonMapper.writeValueAsString(update); + String msg = JacksonUtil.OBJECT_MAPPER.writeValueAsString(update); executor.submit(() -> { try { msgEndpoint.send(sessionRef, cmdId, msg); diff --git a/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java b/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java deleted file mode 100644 index 7ce958ef2d..0000000000 --- a/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.utils; - -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.function.Consumer; - -/** - * This class deduplicate executions of the specified function. - * Useful in cluster mode, when you get event about partition change multiple times. - * Assuming that the function execution is expensive, we should execute it immediately when first time event occurs and - * later, once the processing of first event is done, process last pending task. - * - * @param

parameters of the function - */ -@Slf4j -public class EventDeduplicationExecutor

{ - private final String name; - private final ExecutorService executor; - private final Consumer

function; - private P pendingTask; - private boolean busy; - - public EventDeduplicationExecutor(String name, ExecutorService executor, Consumer

function) { - this.name = name; - this.executor = executor; - this.function = function; - } - - public void submit(P params) { - log.info("[{}] Going to submit: {}", name, params); - synchronized (EventDeduplicationExecutor.this) { - if (!busy) { - busy = true; - pendingTask = null; - try { - log.info("[{}] Submitting task: {}", name, params); - executor.submit(() -> { - try { - log.info("[{}] Executing task: {}", name, params); - function.accept(params); - } catch (Throwable e) { - log.warn("[{}] Failed to process task with parameters: {}", name, params, e); - throw e; - } finally { - unlockAndProcessIfAny(); - } - }); - } catch (Throwable e) { - log.warn("[{}] Failed to submit task with parameters: {}", name, params, e); - unlockAndProcessIfAny(); - throw e; - } - } else { - log.info("[{}] Task is already in progress. {} pending task: {}", name, pendingTask == null ? "adding" : "updating", params); - pendingTask = params; - } - } - } - - private void unlockAndProcessIfAny() { - synchronized (EventDeduplicationExecutor.this) { - busy = false; - if (pendingTask != null) { - submit(pendingTask); - } - } - } -} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5549861950..10c4c9edac 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -73,6 +73,8 @@ server: min_timeout: "${MIN_SERVER_SIDE_RPC_TIMEOUT:5000}" # Default value of the server side RPC timeout. default_timeout: "${DEFAULT_SERVER_SIDE_RPC_TIMEOUT:10000}" + rate_limits: + reset_password_per_user: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:3600}" # Application info app: @@ -571,6 +573,7 @@ spring: password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:16}" + registerMbeans: "${SPRING_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # true - enable MBean to diagnose pools state via JMX # Audit log parameters audit-log: @@ -1209,3 +1212,4 @@ management: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). include: '${METRICS_ENDPOINTS_EXPOSE:info}' + diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 06cee7b71d..32d1f65765 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -52,7 +52,7 @@ public abstract class AbstractControllerTest extends AbstractNotifyEntityTest { @LocalServerPort protected int wsPort; - private TbTestWebSocketClient wsClient; // lazy + private volatile TbTestWebSocketClient wsClient; // lazy public TbTestWebSocketClient getWsClient() { if (wsClient == null) { diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 24ce820760..4afb12e638 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -29,14 +29,19 @@ import lombok.extern.slf4j.Slf4j; import org.hamcrest.Matcher; import org.hibernate.exception.ConstraintViolationException; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -52,6 +57,9 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; @@ -69,6 +77,7 @@ import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -83,7 +92,6 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.config.ThingsboardSecurityConfiguration; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.tenant.TenantProfileService; -import org.thingsboard.server.service.mail.TestMailService; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest; import org.thingsboard.server.service.security.auth.rest.LoginRequest; @@ -98,6 +106,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; @@ -144,6 +153,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected MockMvc mockMvc; + protected String currentActivateToken; + protected String currentResetPasswordToken; + protected String token; protected String refreshToken; protected String username; @@ -168,6 +180,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { @Autowired private TenantProfileService tenantProfileService; + @SpyBean + protected MailService mailService; + @Rule public TestRule watcher = new TestWatcher() { protected void starting(Description description) { @@ -198,7 +213,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { @Before public void setupWebTest() throws Exception { - log.info("Executing web test setup"); + log.debug("Executing web test setup"); + + setupMailServiceMock(); if (this.mockMvc == null) { this.mockMvc = webAppContextSetup(webApplicationContext) @@ -238,12 +255,33 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { resetTokens(); - log.info("Executed web test setup"); + log.debug("Executed web test setup"); + } + + private void setupMailServiceMock() throws ThingsboardException { + Mockito.doNothing().when(mailService).sendAccountActivatedEmail(anyString(), anyString()); + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + String activationLink = (String) args[0]; + currentActivateToken = activationLink.split("=")[1]; + return null; + } + }).when(mailService).sendActivationEmail(anyString(), anyString()); + + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + String passwordResetLink = (String) args[0]; + currentResetPasswordToken = passwordResetLink.split("=")[1]; + return null; + } + }).when(mailService).sendResetPasswordEmailAsync(anyString(), anyString()); } @After public void teardownWebTest() throws Exception { - log.info("Executing web test teardown"); + log.debug("Executing web test teardown"); loginSysAdmin(); doDelete("/api/tenant/" + tenantId.getId().toString()) @@ -368,11 +406,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } private JsonNode getActivateRequest(String password) throws Exception { - doGet("/api/noauth/activate?activateToken={activateToken}", TestMailService.currentActivateToken) + doGet("/api/noauth/activate?activateToken={activateToken}", this.currentActivateToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + TestMailService.currentActivateToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + this.currentActivateToken)); return new ObjectMapper().createObjectNode() - .put("activateToken", TestMailService.currentActivateToken) + .put("activateToken", this.currentActivateToken) .put("password", password); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java index 021171f1af..e6c27aa4a8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java @@ -21,12 +21,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Test; import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.security.model.JwtSettings; -import org.thingsboard.server.service.mail.DefaultMailService; import java.nio.charset.StandardCharsets; import java.util.Base64; @@ -35,6 +32,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -43,12 +42,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. public abstract class BaseAdminControllerTest extends AbstractControllerTest { final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "thingsboardDefaultSigningKey"); - @Autowired - MailService mailService; - - @Autowired - DefaultMailService defaultMailService; - @Test public void testFindAdminSettingsByKey() throws Exception { loginSysAdmin(); @@ -118,10 +111,12 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { @Test public void testSendTestMail() throws Exception { + Mockito.doNothing().when(mailService).sendTestMail(any(), anyString()); loginSysAdmin(); AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); doPost("/api/admin/settings/testMail", adminSettings) .andExpect(status().isOk()); + Mockito.verify(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); } @Test @@ -137,15 +132,8 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { adminSettings.setJsonValue(objectNode); - Mockito.doAnswer((invocations) -> { - var jsonConfig = (JsonNode) invocations.getArgument(0); - var email = (String) invocations.getArgument(1); - - defaultMailService.sendTestMail(jsonConfig, email); - return null; - }).when(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); doPost("/api/admin/settings/testMail", adminSettings).andExpect(status().is5xxServerError()); - Mockito.doNothing().when(mailService).sendTestMail(Mockito.any(), Mockito.any()); + Mockito.verify(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); } void resetJwtSettingsToDefault() throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index f289c8066f..7149f52630 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.user.UserDao; -import org.thingsboard.server.service.mail.TestMailService; import java.util.ArrayList; import java.util.Collections; @@ -54,6 +53,8 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -106,12 +107,12 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); resetTokens(); - doGet("/api/noauth/activate?activateToken={activateToken}", TestMailService.currentActivateToken) + doGet("/api/noauth/activate?activateToken={activateToken}", this.currentActivateToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + TestMailService.currentActivateToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + this.currentActivateToken)); JsonNode activateRequest = new ObjectMapper().createObjectNode() - .put("activateToken", TestMailService.currentActivateToken) + .put("activateToken", this.currentActivateToken) .put("password", "testPassword"); JsonNode tokenInfo = readResponse(doPost("/api/noauth/activate", activateRequest).andExpect(status().isOk()), JsonNode.class); @@ -208,17 +209,19 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { doPost("/api/noauth/resetPasswordByEmail", resetPasswordByEmailRequest) .andExpect(status().isOk()); Thread.sleep(1000); - doGet("/api/noauth/resetPassword?resetToken={resetToken}", TestMailService.currentResetPasswordToken) + doGet("/api/noauth/resetPassword?resetToken={resetToken}", this.currentResetPasswordToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/resetPassword?resetToken=" + TestMailService.currentResetPasswordToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/resetPassword?resetToken=" + this.currentResetPasswordToken)); JsonNode resetPasswordRequest = new ObjectMapper().createObjectNode() - .put("resetToken", TestMailService.currentResetPasswordToken) + .put("resetToken", this.currentResetPasswordToken) .put("password", "testPassword2"); + Mockito.doNothing().when(mailService).sendPasswordWasResetEmail(anyString(), anyString()); JsonNode tokenInfo = readResponse( doPost("/api/noauth/resetPassword", resetPasswordRequest) .andExpect(status().isOk()), JsonNode.class); + Mockito.verify(mailService).sendPasswordWasResetEmail(anyString(), anyString()); validateAndSetJwtToken(tokenInfo, email); doGet("/api/auth/user") diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseWebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseWebsocketApiTest.java index dbadcb271b..5494fee4a2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseWebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseWebsocketApiTest.java @@ -550,7 +550,7 @@ public abstract class BaseWebsocketApiTest extends AbstractControllerTest { SingleEntityFilter entityFilter = new SingleEntityFilter(); entityFilter.setSingleEntity(tenantId); - assertThatNoException().isThrownBy(() -> { + assertThatNoException().as("subscribeForAttributes").isThrownBy(() -> { JsonNode update = getWsClient().subscribeForAttributes(tenantId, TbAttributeSubscriptionScope.SERVER_SCOPE.name(), List.of("attr")); assertThat(update.get("errorMsg").isNull()).isTrue(); assertThat(update.get("errorCode").asInt()).isEqualTo(SubscriptionErrorCode.NO_ERROR.getCode()); @@ -562,7 +562,7 @@ public abstract class BaseWebsocketApiTest extends AbstractControllerTest { new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry("attr", expectedAttrValue)) )); JsonNode update = JacksonUtil.toJsonNode(getWsClient().waitForUpdate()); - assertThat(update).isNotNull(); + assertThat(update).as("waitForUpdate").isNotNull(); assertThat(update.get("data").get("attr").get(0).get(1).asText()).isEqualTo(expectedAttrValue); } @@ -599,15 +599,17 @@ public abstract class BaseWebsocketApiTest extends AbstractControllerTest { tsService.saveAndNotify(device.getTenantId(), null, device.getId(), tsData, 0, new FutureCallback() { @Override public void onSuccess(@Nullable Void result) { + log.debug("sendTelemetry callback onSuccess"); latch.countDown(); } @Override public void onFailure(Throwable t) { + log.error("Failed to send telemetry", t); latch.countDown(); } }); - latch.await(3, TimeUnit.SECONDS); + assertThat(latch.await(TIMEOUT, TimeUnit.SECONDS)).as("await sendTelemetry callback"); } private void sendAttributes(Device device, TbAttributeSubscriptionScope scope, List attrData) throws InterruptedException { @@ -619,14 +621,16 @@ public abstract class BaseWebsocketApiTest extends AbstractControllerTest { tsService.saveAndNotify(tenantId, entityId, scope.name(), attrData, new FutureCallback() { @Override public void onSuccess(@Nullable Void result) { + log.debug("sendAttributes callback onSuccess"); latch.countDown(); } @Override public void onFailure(Throwable t) { + log.error("Failed to sendAttributes", t); latch.countDown(); } }); - latch.await(3, TimeUnit.SECONDS); + assertThat(latch.await(TIMEOUT, TimeUnit.SECONDS)).as("await sendAttributes callback").isTrue(); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java index 56df8513fe..828d279e93 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java @@ -192,7 +192,7 @@ public abstract class BaseWidgetsBundleControllerTest extends AbstractController WidgetsBundle savedWidgetsBundle = doPost("/api/widgetsBundle", widgetsBundle, WidgetsBundle.class); savedWidgetsBundle.setAlias("new_alias"); - Mockito.reset(tbClusterService); + Mockito.clearInvocations(tbClusterService); doPost("/api/widgetsBundle", savedWidgetsBundle) .andExpect(status().isBadRequest()) diff --git a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java index db26e9f6df..1f9ac4ea69 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java @@ -47,6 +47,7 @@ import java.util.concurrent.TimeUnit; @Slf4j public class TbTestWebSocketClient extends WebSocketClient { + private static final long TIMEOUT = TimeUnit.SECONDS.toMillis(30); private volatile String lastMsg; private volatile CountDownLatch reply; private volatile CountDownLatch update; @@ -87,12 +88,14 @@ public class TbTestWebSocketClient extends WebSocketClient { } public void registerWaitForUpdate(int count) { + log.debug("registerWaitForUpdate [{}]", count); lastMsg = null; update = new CountDownLatch(count); } @Override public void send(String text) throws NotYetConnectedException { + log.debug("send [{}]", text); reply = new CountDownLatch(1); super.send(text); } @@ -110,21 +113,31 @@ public class TbTestWebSocketClient extends WebSocketClient { } public String waitForUpdate() { - return waitForUpdate(TimeUnit.SECONDS.toMillis(3)); + return waitForUpdate(TIMEOUT); } public String waitForUpdate(long ms) { + log.debug("waitForUpdate [{}]", ms); try { - update.await(ms, TimeUnit.MILLISECONDS); + if (!update.await(ms, TimeUnit.MILLISECONDS)) { + log.warn("Failed to await update (waiting time [{}]ms elapsed)", ms, new RuntimeException("stacktrace")); + } } catch (InterruptedException e) { - log.warn("Failed to await reply", e); + log.warn("Failed to await update", e); } return lastMsg; } public String waitForReply() { + return waitForReply(TIMEOUT); + } + + public String waitForReply(long ms) { + log.debug("waitForReply [{}]", ms); try { - reply.await(3, TimeUnit.SECONDS); + if (!reply.await(ms, TimeUnit.MILLISECONDS)) { + log.warn("Failed to await reply (waiting time [{}]ms elapsed)", ms, new RuntimeException("stacktrace")); + } } catch (InterruptedException e) { log.warn("Failed to await reply", e); } diff --git a/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java b/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java deleted file mode 100644 index 7c4c45c951..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.mail; - -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Profile; -import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.server.common.data.exception.ThingsboardException; - -@Profile("test") -@Configuration -public class TestMailService { - - public static String currentActivateToken; - public static String currentResetPasswordToken; - - @Bean - @Primary - public MailService mailService() throws ThingsboardException { - MailService mailService = Mockito.mock(MailService.class); - Mockito.doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String activationLink = (String) args[0]; - currentActivateToken = activationLink.split("=")[1]; - return null; - } - }).when(mailService).sendActivationEmail(Mockito.anyString(), Mockito.anyString()); - Mockito.doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String passwordResetLink = (String) args[0]; - currentResetPasswordToken = passwordResetLink.split("=")[1]; - return null; - } - }).when(mailService).sendResetPasswordEmailAsync(Mockito.anyString(), Mockito.anyString()); - return mailService; - } - -} diff --git a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java index 08cd031a52..4373a4c108 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java @@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest @TestPropertySource(properties = { + "js.evaluator=local", "js.max_script_body_size=50", "js.max_total_args_size=50", "js.max_result_size=50", diff --git a/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java index cf4de21cc2..89a4b707bf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java @@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; @Slf4j public abstract class AbstractTransportIntegrationTest extends AbstractControllerTest { - protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 10; + protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 30; protected static final String MQTT_URL = "tcp://localhost:1883"; protected static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/"; diff --git a/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java index 41cc4c2b76..ab2f6b9bca 100644 --- a/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java @@ -15,30 +15,14 @@ */ package org.thingsboard.server.transport; -import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; -import org.junit.BeforeClass; -import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; -import org.thingsboard.server.dao.CustomCassandraCQLUnit; -import org.thingsboard.server.queue.memory.InMemoryStorage; - -import java.util.Arrays; +import org.thingsboard.server.dao.AbstractNoSqlContainer; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({ "org.thingsboard.server.transport.*.telemetry.timeseries.nosql.*Test", }) -public class TransportNoSqlTestSuite { - - @ClassRule - public static CustomCassandraCQLUnit cassandraUnit = - new CustomCassandraCQLUnit( - Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-keyspace.cql", false, false), - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), - new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) - ), - "cassandra-test.yaml", 30000l); +public class TransportNoSqlTestSuite extends AbstractNoSqlContainer { } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index d90af7b6ba..1266ef2c29 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -45,6 +45,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @TestPropertySource(properties = { + "coap.enabled=true", + "service.integrations.supported=ALL", "transport.coap.enabled=true", }) @Slf4j diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java index 0f1815b46b..6cb6e49e79 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java @@ -46,8 +46,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @TestPropertySource(properties = { + "service.integrations.supported=ALL", "transport.mqtt.enabled=true", - "js.evaluator=mock", }) @Slf4j public abstract class AbstractMqttIntegrationTest extends AbstractTransportIntegrationTest { diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index 60af3b66f7..c37f22de36 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.transport.mqtt.mqttv3.attributes; +import com.fasterxml.jackson.core.type.TypeReference; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; @@ -22,6 +23,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.squareup.wire.schema.internal.parser.ProtoFileElement; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DynamicProtoUtils; @@ -45,6 +47,7 @@ import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestClient; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -125,14 +128,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateUpdateAttributesJsonResponse(onUpdateCallback, SHARED_ATTRIBUTES_PAYLOAD); MqttTestCallback onDeleteCallback = new MqttTestCallback(); client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateUpdateAttributesJsonResponse(onDeleteCallback, SHARED_ATTRIBUTES_DELETED_RESPONSE); client.disconnect(); } @@ -145,13 +150,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateUpdateAttributesProtoResponse(onUpdateCallback); MqttTestCallback onDeleteCallback = new MqttTestCallback(); client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateDeleteAttributesProtoResponse(onDeleteCallback); client.disconnect(); } @@ -162,7 +169,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateUpdateAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); List tsKvProtoList = getTsKvProtoList("shared"); attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); @@ -178,7 +185,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateDeleteAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); @@ -209,7 +216,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(GATEWAY_ATTRIBUTES_TOPIC, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateJsonGatewayUpdateAttributesResponse(onUpdateCallback, deviceName, SHARED_ATTRIBUTES_PAYLOAD); @@ -217,7 +225,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateJsonGatewayUpdateAttributesResponse(onDeleteCallback, deviceName, SHARED_ATTRIBUTES_DELETED_RESPONSE); client.disconnect(); @@ -246,7 +255,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName, String expectResultData) { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); assertEquals(JacksonUtil.toJsonNode(getGatewayAttributesResponseJson(deviceName, expectResultData)), JacksonUtil.fromBytes(callback.getPayloadBytes())); } @@ -260,8 +269,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); List tsKvProtoList = getTsKvProtoList("shared"); @@ -285,8 +295,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoGatewayDeleteAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); TransportProtos.AttributeUpdateNotificationMsg attributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); @@ -391,9 +402,19 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt 100); assertNotNull(device); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + + String attributeValuesUrl = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/attributes/CLIENT_SCOPE?keys=" + clientKeysStr; + + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> { + List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() {}); + return attributes.size() == 5; + }); + SingleEntityFilter dtf = new SingleEntityFilter(); dtf.setSingleEntity(device.getId()); - String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; List clientKeysList = List.of(clientKeysStr.split(",")); List sharedKeysList = List.of(sharedKeysStr.split(",")); @@ -538,13 +559,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonResponse(MqttTestCallback callback, String expectedResponse) throws InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(callback.getPayloadBytes())); } protected void validateProtoResponse(MqttTestCallback callback, TransportProtos.GetAttributeResponseMsg expectedResponse) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); assertEquals(expectedResponse.getRequestId(), actualAttributesResponse.getRequestId()); @@ -567,14 +590,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonResponseGateway(MqttTestCallback callback, String deviceName, String expectedValues) throws InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); String expectedRequestPayload = "{\"id\":1,\"device\":\"" + deviceName + "\",\"values\":" + expectedValues + "}"; assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.fromBytes(callback.getPayloadBytes())); } protected void validateProtoClientResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, true); TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); @@ -590,7 +615,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoSharedResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, false); TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); diff --git a/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java b/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java deleted file mode 100644 index ef75657ed4..0000000000 --- a/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.util; - -import com.google.common.util.concurrent.MoreExecutors; -import lombok.extern.slf4j.Slf4j; -import org.junit.After; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.utils.EventDeduplicationExecutor; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class EventDeduplicationExecutorTest { - - ThingsBoardThreadFactory threadFactory = ThingsBoardThreadFactory.forName(getClass().getSimpleName()); - ExecutorService executor; - - @After - public void tearDown() throws Exception { - if (executor != null) { - executor.shutdownNow(); - } - } - - @Test - public void testSimpleFlowSameThread() throws InterruptedException { - simpleFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testPeriodicFlowSameThread() throws InterruptedException { - periodicFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testExceptionFlowSameThread() throws InterruptedException { - exceptionFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testSimpleFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - simpleFlow(executor); - } - - @Test - public void testPeriodicFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - periodicFlow(executor); - } - - @Test - public void testExceptionFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - exceptionFlow(executor); - } - - @Test - public void testSimpleFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - simpleFlow(executor); - } - - @Test - public void testPeriodicFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - periodicFlow(executor); - } - - @Test - public void testExceptionFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - exceptionFlow(executor); - } - - private void simpleFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - executor.submit(params1); - executor.submit(params2); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params1); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - private void periodicFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - executor.submit(params1); - Thread.sleep(500); - executor.submit(params2); - Thread.sleep(500); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params1); - Mockito.verify(function).accept(params2); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - private void exceptionFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - Mockito.doThrow(new RuntimeException()).when(function).accept("params1"); - executor.submit(params1); - executor.submit(params2); - Thread.sleep(500); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params2); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - public static class StringConsumer implements Consumer { - @Override - public void accept(String s) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } - -} diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index e4a36c151d..ad86ff736b 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -1,3 +1,4 @@ +js.evaluator=mock transport.lwm2m.server.security.credentials.enabled=true transport.lwm2m.server.security.credentials.type=KEYSTORE transport.lwm2m.server.security.credentials.keystore.store_file=lwm2m/credentials/lwm2mserver.jks @@ -20,6 +21,9 @@ transport.mqtt.enabled=false transport.coap.enabled=false transport.lwm2m.enabled=false transport.snmp.enabled=false +coap.enabled=false +integrations.rpc.enabled=false +service.integrations.supported=NONE # Low latency settings to perform tests as fast as possible sql.attributes.batch_max_delay=5 diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 3762c8aa7c..23e6d8c2f6 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -13,8 +13,10 @@ - + + + diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index a9f262d6ca..c5af954392 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -108,6 +108,10 @@ public abstract class RedisTbTransactionalCache keys) { + //Redis expects at least 1 key to delete. Otherwise - ERR wrong number of arguments for 'del' command + if (keys.isEmpty()) { + return; + } try (var connection = connectionFactory.getConnection()) { connection.del(keys.stream().map(this::getRawKey).toArray(byte[][]::new)); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 6a2b0a58d6..3b38aa57c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -18,9 +18,14 @@ package org.thingsboard.server.common.data; import com.google.common.base.Splitter; import org.apache.commons.lang3.RandomStringUtils; +import java.security.SecureRandom; +import java.util.Base64; + import static org.apache.commons.lang3.StringUtils.repeat; public class StringUtils { + public static final SecureRandom RANDOM = new SecureRandom(); + public static final String EMPTY = ""; public static final int INDEX_NOT_FOUND = -1; @@ -180,4 +185,11 @@ public class StringUtils { return RandomStringUtils.randomAlphabetic(count); } + public static String generateSafeToken(int length) { + byte[] bytes = new byte[length]; + RANDOM.nextBytes(bytes); + Base64.Encoder encoder = Base64.getUrlEncoder().withoutPadding(); + return encoder.encodeToString(bytes); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 31f81d5510..3fc1626f52 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -68,7 +68,7 @@ public class HashPartitionService implements PartitionService { private final TenantRoutingInfoService tenantRoutingInfoService; private final QueueRoutingInfoService queueRoutingInfoService; - private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); private final ConcurrentMap partitionTopicsMap = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizesMap = new ConcurrentHashMap<>(); @@ -217,17 +217,19 @@ public class HashPartitionService implements PartitionService { } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); - ConcurrentMap> oldPartitions = myPartitions; - myPartitions = new ConcurrentHashMap<>(); + final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i); if (currentService.equals(serviceInfo)) { - myPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); + newPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); } } }); + final ConcurrentMap> oldPartitions = myPartitions; + myPartitions = newPartitions; + oldPartitions.forEach((queueKey, partitions) -> { if (!myPartitions.containsKey(queueKey)) { log.info("[{}] NO MORE PARTITIONS FOR CURRENT KEY", queueKey); diff --git a/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 82c806ad4f..8fff7fc0fe 100644 --- a/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -109,17 +109,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final Lock scriptsLock = new ReentrantLock(); @PostConstruct + @Override public void init() { super.init(); requestTemplate.init(); } @PreDestroy - public void destroy() { + @Override + public void stop() { super.stop(); if (requestTemplate != null) { requestTemplate.stop(); } + callbackExecutor.shutdownNow(); } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java index c995474452..9e3ae3c032 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java @@ -104,6 +104,7 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { } @PostConstruct + @Override public void init() { super.init(); jsExecutor = MoreExecutors.listeningDecorator(Executors.newWorkStealingPool(jsExecutorThreadPoolSize)); @@ -122,11 +123,15 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { } @PreDestroy + @Override public void stop() { super.stop(); if (monitorExecutorService != null) { monitorExecutorService.shutdownNow(); } + if (jsExecutor != null) { + jsExecutor.shutdownNow(); + } } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 7d2043557e..f98e33b615 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -119,6 +119,7 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem @SneakyThrows @PostConstruct + @Override public void init() { super.init(); OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); @@ -142,7 +143,9 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem } @PreDestroy - public void destroy() { + @Override + public void stop() { + super.stop(); if (executor != null) { executor.shutdownNow(); } diff --git a/dao/pom.xml b/dao/pom.xml index 30dc1adcf0..4fc8a3c08d 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -162,26 +162,6 @@ com.google.guava guava - - org.cassandraunit - cassandra-unit - - - org.slf4j - slf4j-log4j12 - - - org.hibernate - hibernate-validator - - - test - - - org.apache.cassandra - cassandra-thrift - test - com.google.protobuf protobuf-java @@ -211,6 +191,11 @@ spring-test test + + org.testcontainers + cassandra + test + org.testcontainers postgresql diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 584ae84531..017fe4315c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -66,7 +66,7 @@ public class CachedAttributesService implements AttributesService { private final TbTransactionalCache cache; private ListeningExecutorService cacheExecutor; - @Value("${cache.type}") + @Value("${cache.type:caffeine}") private String cacheType; public CachedAttributesService(AttributesDao attributesDao, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index f8664331cd..68c11d9d2b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -806,6 +806,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private String entityNameQuery(QueryContext ctx, EntityNameFilter filter) { ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter()); + + if (filter.getEntityNameFilter().startsWith("%") || filter.getEntityNameFilter().endsWith("%")) { + return "lower(e.search_text) like lower(:entity_filter_name_filter)"; + } + return "lower(e.search_text) like lower(concat(:entity_filter_name_filter, '%%'))"; } @@ -840,6 +845,9 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { ctx.addStringListParameter("entity_filter_type_query_types", types); if (!StringUtils.isEmpty(name)) { ctx.addStringParameter("entity_filter_type_query_name", name); + if (name.startsWith("%") || name.endsWith("%")) { + return typesFilter + " and lower(e.search_text) like lower(:entity_filter_type_query_name)"; + } return typesFilter + " and lower(e.search_text) like lower(concat(:entity_filter_type_query_name, '%%'))"; } else { return typesFilter; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index afffdbb0cc..b58f0dc7c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -228,7 +228,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme long ts = latest.getTs(); ListenableFuture removedLatestFuture; - if (ts > query.getStartTs() && ts <= query.getEndTs()) { + if (ts >= query.getStartTs() && ts < query.getEndTs()) { TsKvLatestEntity latestEntity = new TsKvLatestEntity(); latestEntity.setEntityId(entityId.getId()); latestEntity.setKey(getOrSaveKeyId(query.getKey())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index c8c5175e5f..b4ad62c0fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -29,7 +29,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -40,7 +39,6 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.common.data.security.UserSettings; import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.IncorrectParameterException; @@ -51,6 +49,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import static org.thingsboard.server.common.data.StringUtils.generateSafeToken; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; import static org.thingsboard.server.dao.service.Validator.validateString; @@ -126,7 +125,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic if (user.getId() == null) { UserCredentials userCredentials = new UserCredentials(); userCredentials.setEnabled(false); - userCredentials.setActivateToken(StringUtils.randomAlphanumeric(DEFAULT_TOKEN_LENGTH)); + userCredentials.setActivateToken(generateSafeToken(DEFAULT_TOKEN_LENGTH)); userCredentials.setUserId(new UserId(savedUser.getUuidId())); saveUserCredentialsAndPasswordHistory(user.getTenantId(), userCredentials); } @@ -192,7 +191,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic if (!userCredentials.isEnabled()) { throw new DisabledException(String.format("User credentials not enabled [%s]", email)); } - userCredentials.setResetToken(StringUtils.randomAlphanumeric(DEFAULT_TOKEN_LENGTH)); + userCredentials.setResetToken(generateSafeToken(DEFAULT_TOKEN_LENGTH)); return saveUserCredentials(tenantId, userCredentials); } @@ -202,7 +201,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic if (!userCredentials.isEnabled()) { throw new IncorrectParameterException("Unable to reset password for inactive user"); } - userCredentials.setResetToken(StringUtils.randomAlphanumeric(DEFAULT_TOKEN_LENGTH)); + userCredentials.setResetToken(generateSafeToken(DEFAULT_TOKEN_LENGTH)); return saveUserCredentials(tenantId, userCredentials); } diff --git a/dao/src/main/resources/cassandra/schema-ts-latest.cql b/dao/src/main/resources/cassandra/schema-ts-latest.cql index e3c72ab6da..6b76c6187c 100644 --- a/dao/src/main/resources/cassandra/schema-ts-latest.cql +++ b/dao/src/main/resources/cassandra/schema-ts-latest.cql @@ -15,7 +15,7 @@ -- CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf ( - entity_type text, // (DEVICE, CUSTOMER, TENANT) + entity_type text, -- (DEVICE, CUSTOMER, TENANT) entity_id timeuuid, key text, ts bigint, diff --git a/dao/src/main/resources/cassandra/schema-ts.cql b/dao/src/main/resources/cassandra/schema-ts.cql index fb60ed9e78..ae8c11a703 100644 --- a/dao/src/main/resources/cassandra/schema-ts.cql +++ b/dao/src/main/resources/cassandra/schema-ts.cql @@ -15,7 +15,7 @@ -- CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf ( - entity_type text, // (DEVICE, CUSTOMER, TENANT) + entity_type text, -- (DEVICE, CUSTOMER, TENANT) entity_id timeuuid, key text, partition bigint, @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf ( ); CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf ( - entity_type text, // (DEVICE, CUSTOMER, TENANT) + entity_type text, -- (DEVICE, CUSTOMER, TENANT) entity_id timeuuid, key text, partition bigint, diff --git a/dao/src/test/java/org/apache/cassandra/io/sstable/Descriptor.java b/dao/src/test/java/org/apache/cassandra/io/sstable/Descriptor.java deleted file mode 100644 index 69a164f729..0000000000 --- a/dao/src/test/java/org/apache/cassandra/io/sstable/Descriptor.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable; - -import java.io.File; -import java.io.IOError; -import java.io.IOException; -import java.util.*; -import java.util.regex.Pattern; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.CharMatcher; -import com.google.common.base.Objects; - -import org.apache.cassandra.db.Directories; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer; -import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer; -import org.apache.cassandra.io.sstable.metadata.MetadataSerializer; -import org.apache.cassandra.utils.Pair; - -import static org.apache.cassandra.io.sstable.Component.separator; - -/** - * A SSTable is described by the keyspace and column family it contains data - * for, a generation (where higher generations contain more recent data) and - * an alphabetic version string. - * - * A descriptor can be marked as temporary, which influences generated filenames. - */ -public class Descriptor -{ - public static String TMP_EXT = ".tmp"; - - /** canonicalized path to the directory where SSTable resides */ - public final File directory; - /** version has the following format: [a-z]+ */ - public final Version version; - public final String ksname; - public final String cfname; - public final int generation; - public final SSTableFormat.Type formatType; - /** digest component - might be {@code null} for old, legacy sstables */ - public final Component digestComponent; - private final int hashCode; - - /** - * A descriptor that assumes CURRENT_VERSION. - */ - @VisibleForTesting - public Descriptor(File directory, String ksname, String cfname, int generation) - { - this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, generation, SSTableFormat.Type.current(), null); - } - - /** - * Constructor for sstable writers only. - */ - public Descriptor(File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType) - { - this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType, Component.digestFor(formatType.info.getLatestVersion().uncompressedChecksumType())); - } - - @VisibleForTesting - public Descriptor(String version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType) - { - this(formatType.info.getVersion(version), directory, ksname, cfname, generation, formatType, Component.digestFor(formatType.info.getLatestVersion().uncompressedChecksumType())); - } - - public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType, Component digestComponent) - { - assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass()); - this.version = version; - try - { - this.directory = directory.getCanonicalFile(); - } - catch (IOException e) - { - throw new IOError(e); - } - this.ksname = ksname; - this.cfname = cfname; - this.generation = generation; - this.formatType = formatType; - this.digestComponent = digestComponent; - - hashCode = Objects.hashCode(version, this.directory, generation, ksname, cfname, formatType); - } - - public Descriptor withGeneration(int newGeneration) - { - return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType, digestComponent); - } - - public Descriptor withFormatType(SSTableFormat.Type newType) - { - return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType, digestComponent); - } - - public Descriptor withDigestComponent(Component newDigestComponent) - { - return new Descriptor(version, directory, ksname, cfname, generation, formatType, newDigestComponent); - } - - public String tmpFilenameFor(Component component) - { - return filenameFor(component) + TMP_EXT; - } - - public String filenameFor(Component component) - { - return baseFilename() + separator + component.name(); - } - - public String baseFilename() - { - StringBuilder buff = new StringBuilder(); - buff.append(directory).append(File.separatorChar); - appendFileName(buff); - return buff.toString(); - } - - private void appendFileName(StringBuilder buff) - { - if (!version.hasNewFileName()) - { - buff.append(ksname).append(separator); - buff.append(cfname).append(separator); - } - buff.append(version).append(separator); - buff.append(generation); - if (formatType != SSTableFormat.Type.LEGACY) - buff.append(separator).append(formatType.name); - } - - public String relativeFilenameFor(Component component) - { - final StringBuilder buff = new StringBuilder(); - appendFileName(buff); - buff.append(separator).append(component.name()); - return buff.toString(); - } - - public SSTableFormat getFormat() - { - return formatType.info; - } - - /** Return any temporary files found in the directory */ - public List getTemporaryFiles() - { - List ret = new ArrayList<>(); - File[] tmpFiles = directory.listFiles((dir, name) -> - name.endsWith(Descriptor.TMP_EXT)); - - for (File tmpFile : tmpFiles) - ret.add(tmpFile); - - return ret; - } - - /** - * Files obsoleted by CASSANDRA-7066 : temporary files and compactions_in_progress. We support - * versions 2.1 (ka) and 2.2 (la). - * Temporary files have tmp- or tmplink- at the beginning for 2.2 sstables or after ks-cf- for 2.1 sstables - */ - - private final static String LEGACY_COMP_IN_PROG_REGEX_STR = "^compactions_in_progress(\\-[\\d,a-f]{32})?$"; - private final static Pattern LEGACY_COMP_IN_PROG_REGEX = Pattern.compile(LEGACY_COMP_IN_PROG_REGEX_STR); - private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$"; - private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR); - - public static boolean isLegacyFile(File file) - { - if (file.isDirectory()) - return file.getParentFile() != null && - file.getParentFile().getName().equalsIgnoreCase("system") && - LEGACY_COMP_IN_PROG_REGEX.matcher(file.getName()).matches(); - else - return LEGACY_TMP_REGEX.matcher(file.getName()).matches(); - } - - public static boolean isValidFile(String fileName) - { - return fileName.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(fileName).matches(); - } - - /** - * @see #fromFilename(File directory, String name) - * @param filename The SSTable filename - * @return Descriptor of the SSTable initialized from filename - */ - public static Descriptor fromFilename(String filename) - { - return fromFilename(filename, false); - } - - public static Descriptor fromFilename(String filename, SSTableFormat.Type formatType) - { - return fromFilename(filename).withFormatType(formatType); - } - - public static Descriptor fromFilename(String filename, boolean skipComponent) - { - File file = new File(filename).getAbsoluteFile(); - return fromFilename(file.getParentFile(), file.getName(), skipComponent).left; - } - - public static Pair fromFilename(File directory, String name) - { - return fromFilename(directory, name, false); - } - - /** - * Filename of the form is vary by version: - * - *

    - *
  • <ksname>-<cfname>-(tmp-)?<version>-<gen>-<component> for cassandra 2.0 and before
    • - *
  • (<tmp marker>-)?<version>-<gen>-<component> for cassandra 3.0 and later
    • - *
- * - * If this is for SSTable of secondary index, directory should ends with index name for 2.1+. - * - * @param directory The directory of the SSTable files - * @param name The name of the SSTable file - * @param skipComponent true if the name param should not be parsed for a component tag - * - * @return A Descriptor for the SSTable, and the Component remainder. - */ - @SuppressWarnings("deprecation") - public static Pair fromFilename(File directory, String name, boolean skipComponent) - { - File parentDirectory = directory != null ? directory : new File("."); - - // tokenize the filename - StringTokenizer st = new StringTokenizer(name, String.valueOf(separator)); - String nexttok; - - // read tokens backwards to determine version - Deque tokenStack = new ArrayDeque<>(); - while (st.hasMoreTokens()) - { - tokenStack.push(st.nextToken()); - } - - // component suffix - String component = skipComponent ? null : tokenStack.pop(); - - nexttok = tokenStack.pop(); - // generation OR format type - SSTableFormat.Type fmt = SSTableFormat.Type.LEGACY; - if (!CharMatcher.digit().matchesAllOf(nexttok)) - { - fmt = SSTableFormat.Type.validate(nexttok); - nexttok = tokenStack.pop(); - } - - // generation - int generation = Integer.parseInt(nexttok); - - // version - nexttok = tokenStack.pop(); - - if (!Version.validate(nexttok)) - throw new UnsupportedOperationException("SSTable " + name + " is too old to open. Upgrade to 2.0 first, and run upgradesstables"); - - Version version = fmt.info.getVersion(nexttok); - - // ks/cf names - String ksname, cfname; - if (version.hasNewFileName()) - { - // for 2.1+ read ks and cf names from directory - File cfDirectory = parentDirectory; - // check if this is secondary index - String indexName = ""; - if (cfDirectory.getName().startsWith(Directories.SECONDARY_INDEX_NAME_SEPARATOR)) - { - indexName = cfDirectory.getName(); - cfDirectory = cfDirectory.getParentFile(); - } - if (cfDirectory.getName().equals(Directories.BACKUPS_SUBDIR)) - { - cfDirectory = cfDirectory.getParentFile(); - } - else if (cfDirectory.getParentFile().getName().equals(Directories.SNAPSHOT_SUBDIR)) - { - cfDirectory = cfDirectory.getParentFile().getParentFile(); - } - cfname = cfDirectory.getName().split("-")[0] + indexName; - ksname = cfDirectory.getParentFile().getName(); - } - else - { - cfname = tokenStack.pop(); - ksname = tokenStack.pop(); - } - assert tokenStack.isEmpty() : "Invalid file name " + name + " in " + directory; - - return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, fmt, - // _assume_ version from version - Component.digestFor(version.uncompressedChecksumType())), - component); - } - - @SuppressWarnings("deprecation") - public IMetadataSerializer getMetadataSerializer() - { - if (version.hasNewStatsFile()) - return new MetadataSerializer(); - else - return new LegacyMetadataSerializer(); - } - - /** - * @return true if the current Cassandra version can read the given sstable version - */ - public boolean isCompatible() - { - return version.isCompatible(); - } - - @Override - public String toString() - { - return baseFilename(); - } - - @Override - public boolean equals(Object o) - { - if (o == this) - return true; - if (!(o instanceof Descriptor)) - return false; - Descriptor that = (Descriptor)o; - return that.directory.equals(this.directory) - && that.generation == this.generation - && that.ksname.equals(this.ksname) - && that.cfname.equals(this.cfname) - && that.formatType == this.formatType; - } - - @Override - public int hashCode() - { - return hashCode; - } -} diff --git a/dao/src/test/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java b/dao/src/test/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java deleted file mode 100644 index 350d27591f..0000000000 --- a/dao/src/test/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable.format; - -import com.google.common.base.CharMatcher; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.io.sstable.format.big.BigFormat; - -/** - * Provides the accessors to data on disk. - */ -public interface SSTableFormat -{ - static boolean enableSSTableDevelopmentTestMode = Boolean.getBoolean("cassandra.test.sstableformatdevelopment"); - - - Version getLatestVersion(); - Version getVersion(String version); - - SSTableWriter.Factory getWriterFactory(); - SSTableReader.Factory getReaderFactory(); - - RowIndexEntry.IndexSerializer getIndexSerializer(CFMetaData cfm, Version version, SerializationHeader header); - - public static enum Type - { - //Used internally to refer to files with no - //format flag in the filename - LEGACY("big", BigFormat.instance), - - //The original sstable format - BIG("big", BigFormat.instance); - - public final SSTableFormat info; - public final String name; - - public static Type current() - { - return BIG; - } - - @SuppressWarnings("deprecation") - private Type(String name, SSTableFormat info) - { - //Since format comes right after generation - //we disallow formats with numeric names - // We have removed this check for compatibility with the embedded cassandra used for tests. - assert !CharMatcher.digit().matchesAllOf(name); - - this.name = name; - this.info = info; - } - - public static Type validate(String name) - { - for (Type valid : Type.values()) - { - //This is used internally for old sstables - if (valid == LEGACY) - continue; - - if (valid.name.equalsIgnoreCase(name)) - return valid; - } - - throw new IllegalArgumentException("No Type constant " + name); - } - } -} diff --git a/dao/src/test/java/org/apache/cassandra/io/util/FileUtils.java b/dao/src/test/java/org/apache/cassandra/io/util/FileUtils.java deleted file mode 100644 index 2609a2c3d0..0000000000 --- a/dao/src/test/java/org/apache/cassandra/io/util/FileUtils.java +++ /dev/null @@ -1,760 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.util; - -import java.io.*; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileAttributeView; -import java.nio.file.attribute.FileStoreAttributeView; -import java.text.DecimalFormat; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.stream.StreamSupport; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.concurrent.ScheduledExecutors; -import org.apache.cassandra.io.FSError; -import org.apache.cassandra.io.FSErrorHandler; -import org.apache.cassandra.io.FSReadError; -import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.utils.JVMStabilityInspector; - -import static com.google.common.base.Throwables.throwIfUnchecked; -import static org.apache.cassandra.utils.Throwables.maybeFail; -import static org.apache.cassandra.utils.Throwables.merge; - -public final class FileUtils -{ - public static final Charset CHARSET = StandardCharsets.UTF_8; - - private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); - public static final long ONE_KB = 1024; - public static final long ONE_MB = 1024 * ONE_KB; - public static final long ONE_GB = 1024 * ONE_MB; - public static final long ONE_TB = 1024 * ONE_GB; - - private static final DecimalFormat df = new DecimalFormat("#.##"); - public static final boolean isCleanerAvailable = false; - private static final AtomicReference> fsErrorHandler = new AtomicReference<>(Optional.empty()); - - public static void createHardLink(String from, String to) - { - createHardLink(new File(from), new File(to)); - } - - public static void createHardLink(File from, File to) - { - if (to.exists()) - throw new RuntimeException("Tried to create duplicate hard link to " + to); - if (!from.exists()) - throw new RuntimeException("Tried to hard link to file that does not exist " + from); - - try - { - Files.createLink(to.toPath(), from.toPath()); - } - catch (IOException e) - { - throw new FSWriteError(e, to); - } - } - - public static File createTempFile(String prefix, String suffix, File directory) - { - try - { - return File.createTempFile(prefix, suffix, directory); - } - catch (IOException e) - { - throw new FSWriteError(e, directory); - } - } - - public static File createTempFile(String prefix, String suffix) - { - return createTempFile(prefix, suffix, new File(System.getProperty("java.io.tmpdir"))); - } - - public static Throwable deleteWithConfirm(String filePath, boolean expect, Throwable accumulate) - { - return deleteWithConfirm(new File(filePath), expect, accumulate); - } - - public static Throwable deleteWithConfirm(File file, boolean expect, Throwable accumulate) - { - boolean exists = file.exists(); - assert exists || !expect : "attempted to delete non-existing file " + file.getName(); - try - { - if (exists) - Files.delete(file.toPath()); - } - catch (Throwable t) - { - try - { - throw new FSWriteError(t, file); - } - catch (Throwable t2) - { - accumulate = merge(accumulate, t2); - } - } - return accumulate; - } - - public static void deleteWithConfirm(String file) - { - deleteWithConfirm(new File(file)); - } - - public static void deleteWithConfirm(File file) - { - maybeFail(deleteWithConfirm(file, true, null)); - } - - public static void renameWithOutConfirm(String from, String to) - { - try - { - atomicMoveWithFallback(new File(from).toPath(), new File(to).toPath()); - } - catch (IOException e) - { - if (logger.isTraceEnabled()) - logger.trace("Could not move file "+from+" to "+to, e); - } - } - - public static void renameWithConfirm(String from, String to) - { - renameWithConfirm(new File(from), new File(to)); - } - - public static void renameWithConfirm(File from, File to) - { - assert from.exists(); - if (logger.isTraceEnabled()) - logger.trace("Renaming {} to {}", from.getPath(), to.getPath()); - // this is not FSWE because usually when we see it it's because we didn't close the file before renaming it, - // and Windows is picky about that. - try - { - atomicMoveWithFallback(from.toPath(), to.toPath()); - } - catch (IOException e) - { - throw new RuntimeException(String.format("Failed to rename %s to %s", from.getPath(), to.getPath()), e); - } - } - - /** - * Move a file atomically, if it fails, it falls back to a non-atomic operation - * @param from - * @param to - * @throws IOException - */ - private static void atomicMoveWithFallback(Path from, Path to) throws IOException - { - try - { - Files.move(from, to, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - } - catch (AtomicMoveNotSupportedException e) - { - logger.trace("Could not do an atomic move", e); - Files.move(from, to, StandardCopyOption.REPLACE_EXISTING); - } - - } - public static void truncate(String path, long size) - { - try(FileChannel channel = FileChannel.open(Paths.get(path), StandardOpenOption.READ, StandardOpenOption.WRITE)) - { - channel.truncate(size); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - public static void closeQuietly(Closeable c) - { - try - { - if (c != null) - c.close(); - } - catch (Exception e) - { - logger.warn("Failed closing {}", c, e); - } - } - - public static void closeQuietly(AutoCloseable c) - { - try - { - if (c != null) - c.close(); - } - catch (Exception e) - { - logger.warn("Failed closing {}", c, e); - } - } - - public static void close(Closeable... cs) throws IOException - { - close(Arrays.asList(cs)); - } - - public static void close(Iterable cs) throws IOException - { - Throwable e = null; - for (Closeable c : cs) - { - try - { - if (c != null) - c.close(); - } - catch (Throwable ex) - { - if (e == null) e = ex; - else e.addSuppressed(ex); - logger.warn("Failed closing stream {}", c, ex); - } - } - maybeFail(e, IOException.class); - } - - public static void closeQuietly(Iterable cs) - { - for (AutoCloseable c : cs) - { - try - { - if (c != null) - c.close(); - } - catch (Exception ex) - { - logger.warn("Failed closing {}", c, ex); - } - } - } - - public static String getCanonicalPath(String filename) - { - try - { - return new File(filename).getCanonicalPath(); - } - catch (IOException e) - { - throw new FSReadError(e, filename); - } - } - - public static String getCanonicalPath(File file) - { - try - { - return file.getCanonicalPath(); - } - catch (IOException e) - { - throw new FSReadError(e, file); - } - } - - /** Return true if file is contained in folder */ - public static boolean isContained(File folder, File file) - { - Path folderPath = Paths.get(getCanonicalPath(folder)); - Path filePath = Paths.get(getCanonicalPath(file)); - - return filePath.startsWith(folderPath); - } - - /** Convert absolute path into a path relative to the base path */ - public static String getRelativePath(String basePath, String path) - { - try - { - return Paths.get(basePath).relativize(Paths.get(path)).toString(); - } - catch(Exception ex) - { - String absDataPath = FileUtils.getCanonicalPath(basePath); - return Paths.get(absDataPath).relativize(Paths.get(path)).toString(); - } - } - - public static void clean(ByteBuffer buffer) - { - if (buffer == null) - return; - } - - public static void createDirectory(String directory) - { - createDirectory(new File(directory)); - } - - public static void createDirectory(File directory) - { - if (!directory.exists()) - { - if (!directory.mkdirs()) - throw new FSWriteError(new IOException("Failed to mkdirs " + directory), directory); - } - } - - public static boolean delete(String file) - { - File f = new File(file); - return f.delete(); - } - - public static void delete(File... files) - { - if (files == null) - { - // CASSANDRA-13389: some callers use Files.listFiles() which, on error, silently returns null - logger.debug("Received null list of files to delete"); - return; - } - - for ( File file : files ) - { - file.delete(); - } - } - - public static void deleteAsync(final String file) - { - Runnable runnable = new Runnable() - { - public void run() - { - deleteWithConfirm(new File(file)); - } - }; - ScheduledExecutors.nonPeriodicTasks.execute(runnable); - } - - public static void visitDirectory(Path dir, Predicate filter, Consumer consumer) - { - try (DirectoryStream stream = Files.newDirectoryStream(dir)) - { - StreamSupport.stream(stream.spliterator(), false) - .map(Path::toFile) - // stream directories are weakly consistent so we always check if the file still exists - .filter(f -> f.exists() && (filter == null || filter.test(f))) - .forEach(consumer); - } - catch (IOException|DirectoryIteratorException ex) - { - logger.error("Failed to list files in {} with exception: {}", dir, ex.getMessage(), ex); - } - } - - public static String stringifyFileSize(double value) - { - double d; - if ( value >= ONE_TB ) - { - d = value / ONE_TB; - String val = df.format(d); - return val + " TiB"; - } - else if ( value >= ONE_GB ) - { - d = value / ONE_GB; - String val = df.format(d); - return val + " GiB"; - } - else if ( value >= ONE_MB ) - { - d = value / ONE_MB; - String val = df.format(d); - return val + " MiB"; - } - else if ( value >= ONE_KB ) - { - d = value / ONE_KB; - String val = df.format(d); - return val + " KiB"; - } - else - { - String val = df.format(value); - return val + " bytes"; - } - } - - /** - * Deletes all files and subdirectories under "dir". - * @param dir Directory to be deleted - * @throws FSWriteError if any part of the tree cannot be deleted - */ - public static void deleteRecursive(File dir) - { - if (dir.isDirectory()) - { - String[] children = dir.list(); - for (String child : children) - deleteRecursive(new File(dir, child)); - } - - // The directory is now empty so now it can be smoked - deleteWithConfirm(dir); - } - - /** - * Schedules deletion of all file and subdirectories under "dir" on JVM shutdown. - * @param dir Directory to be deleted - */ - public static void deleteRecursiveOnExit(File dir) - { - if (dir.isDirectory()) - { - String[] children = dir.list(); - for (String child : children) - deleteRecursiveOnExit(new File(dir, child)); - } - - logger.trace("Scheduling deferred deletion of file: {}", dir); - dir.deleteOnExit(); - } - - public static void handleCorruptSSTable(CorruptSSTableException e) - { - fsErrorHandler.get().ifPresent(handler -> handler.handleCorruptSSTable(e)); - } - - public static void handleFSError(FSError e) - { - fsErrorHandler.get().ifPresent(handler -> handler.handleFSError(e)); - } - - /** - * handleFSErrorAndPropagate will invoke the disk failure policy error handler, - * which may or may not stop the daemon or transports. However, if we don't exit, - * we still want to propagate the exception to the caller in case they have custom - * exception handling - * - * @param e A filesystem error - */ - public static void handleFSErrorAndPropagate(FSError e) - { - JVMStabilityInspector.inspectThrowable(e); - throwIfUnchecked(e); - throw new RuntimeException(e); - } - - /** - * Get the size of a directory in bytes - * @param folder The directory for which we need size. - * @return The size of the directory - */ - public static long folderSize(File folder) - { - final long [] sizeArr = {0L}; - try - { - Files.walkFileTree(folder.toPath(), new SimpleFileVisitor() - { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - { - sizeArr[0] += attrs.size(); - return FileVisitResult.CONTINUE; - } - }); - } - catch (IOException e) - { - logger.error("Error while getting {} folder size. {}", folder, e); - } - return sizeArr[0]; - } - - public static void copyTo(DataInput in, OutputStream out, int length) throws IOException - { - byte[] buffer = new byte[64 * 1024]; - int copiedBytes = 0; - - while (copiedBytes + buffer.length < length) - { - in.readFully(buffer); - out.write(buffer); - copiedBytes += buffer.length; - } - - if (copiedBytes < length) - { - int left = length - copiedBytes; - in.readFully(buffer, 0, left); - out.write(buffer, 0, left); - } - } - - public static boolean isSubDirectory(File parent, File child) throws IOException - { - parent = parent.getCanonicalFile(); - child = child.getCanonicalFile(); - - File toCheck = child; - while (toCheck != null) - { - if (parent.equals(toCheck)) - return true; - toCheck = toCheck.getParentFile(); - } - return false; - } - - public static void append(File file, String ... lines) - { - if (file.exists()) - write(file, Arrays.asList(lines), StandardOpenOption.APPEND); - else - write(file, Arrays.asList(lines), StandardOpenOption.CREATE); - } - - public static void appendAndSync(File file, String ... lines) - { - if (file.exists()) - write(file, Arrays.asList(lines), StandardOpenOption.APPEND, StandardOpenOption.SYNC); - else - write(file, Arrays.asList(lines), StandardOpenOption.CREATE, StandardOpenOption.SYNC); - } - - public static void replace(File file, String ... lines) - { - write(file, Arrays.asList(lines), StandardOpenOption.TRUNCATE_EXISTING); - } - - public static void write(File file, List lines, StandardOpenOption ... options) - { - try - { - Files.write(file.toPath(), - lines, - CHARSET, - options); - } - catch (IOException ex) - { - throw new RuntimeException(ex); - } - } - - public static List readLines(File file) - { - try - { - return Files.readAllLines(file.toPath(), CHARSET); - } - catch (IOException ex) - { - if (ex instanceof NoSuchFileException) - return Collections.emptyList(); - - throw new RuntimeException(ex); - } - } - - public static void setFSErrorHandler(FSErrorHandler handler) - { - fsErrorHandler.getAndSet(Optional.ofNullable(handler)); - } - - /** - * Returns the size of the specified partition. - *

This method handles large file system by returning {@code Long.MAX_VALUE} if the size overflow. - * See JDK-8179320 for more information.

- * - * @param file the partition - * @return the size, in bytes, of the partition or {@code 0L} if the abstract pathname does not name a partition - */ - public static long getTotalSpace(File file) - { - return handleLargeFileSystem(file.getTotalSpace()); - } - - /** - * Returns the number of unallocated bytes on the specified partition. - *

This method handles large file system by returning {@code Long.MAX_VALUE} if the number of unallocated bytes - * overflow. See JDK-8179320 for more information

- * - * @param file the partition - * @return the number of unallocated bytes on the partition or {@code 0L} - * if the abstract pathname does not name a partition. - */ - public static long getFreeSpace(File file) - { - return handleLargeFileSystem(file.getFreeSpace()); - } - - /** - * Returns the number of available bytes on the specified partition. - *

This method handles large file system by returning {@code Long.MAX_VALUE} if the number of available bytes - * overflow. See JDK-8179320 for more information

- * - * @param file the partition - * @return the number of available bytes on the partition or {@code 0L} - * if the abstract pathname does not name a partition. - */ - public static long getUsableSpace(File file) - { - return handleLargeFileSystem(file.getUsableSpace()); - } - - /** - * Returns the {@link FileStore} representing the file store where a file - * is located. This {@link FileStore} handles large file system by returning {@code Long.MAX_VALUE} - * from {@code FileStore#getTotalSpace()}, {@code FileStore#getUnallocatedSpace()} and {@code FileStore#getUsableSpace()} - * it the value is bigger than {@code Long.MAX_VALUE}. See JDK-8162520 - * for more information. - * - * @param path the path to the file - * @return the file store where the file is stored - */ - public static FileStore getFileStore(Path path) throws IOException - { - return new SafeFileStore(Files.getFileStore(path)); - } - - /** - * Handle large file system by returning {@code Long.MAX_VALUE} when the size overflows. - * @param size returned by the Java's FileStore methods - * @return the size or {@code Long.MAX_VALUE} if the size was bigger than {@code Long.MAX_VALUE} - */ - private static long handleLargeFileSystem(long size) - { - return size < 0 ? Long.MAX_VALUE : size; - } - - /** - * Private constructor as the class contains only static methods. - */ - private FileUtils() - { - } - - /** - * FileStore decorator used to safely handle large file system. - * - *

Java's FileStore methods (getTotalSpace/getUnallocatedSpace/getUsableSpace) are limited to reporting bytes as - * signed long (2^63-1), if the filesystem is any bigger, then the size overflows. {@code SafeFileStore} will - * return {@code Long.MAX_VALUE} if the size overflow.

- * - * @see https://bugs.openjdk.java.net/browse/JDK-8162520. - */ - private static final class SafeFileStore extends FileStore - { - /** - * The decorated {@code FileStore} - */ - private final FileStore fileStore; - - public SafeFileStore(FileStore fileStore) - { - this.fileStore = fileStore; - } - - @Override - public String name() - { - return fileStore.name(); - } - - @Override - public String type() - { - return fileStore.type(); - } - - @Override - public boolean isReadOnly() - { - return fileStore.isReadOnly(); - } - - @Override - public long getTotalSpace() throws IOException - { - return handleLargeFileSystem(fileStore.getTotalSpace()); - } - - @Override - public long getUsableSpace() throws IOException - { - return handleLargeFileSystem(fileStore.getUsableSpace()); - } - - @Override - public long getUnallocatedSpace() throws IOException - { - return handleLargeFileSystem(fileStore.getUnallocatedSpace()); - } - - @Override - public boolean supportsFileAttributeView(Class type) - { - return fileStore.supportsFileAttributeView(type); - } - - @Override - public boolean supportsFileAttributeView(String name) - { - return fileStore.supportsFileAttributeView(name); - } - - @Override - public V getFileStoreAttributeView(Class type) - { - return fileStore.getFileStoreAttributeView(type); - } - - @Override - public Object getAttribute(String attribute) throws IOException - { - return fileStore.getAttribute(attribute); - } - } -} diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractNoSqlContainer.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractNoSqlContainer.java new file mode 100644 index 0000000000..3f7c2c779a --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractNoSqlContainer.java @@ -0,0 +1,96 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.thingsboard.server.dao; + +import com.github.dockerjava.api.command.InspectContainerResponse; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.junit.ClassRule; +import org.junit.rules.ExternalResource; +import org.testcontainers.containers.CassandraContainer; +import org.testcontainers.containers.delegate.CassandraDatabaseDelegate; +import org.testcontainers.delegate.DatabaseDelegate; +import org.testcontainers.ext.ScriptUtils; + +import javax.script.ScriptException; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.List; + +@Slf4j +public abstract class AbstractNoSqlContainer { + + public static final List INIT_SCRIPTS = List.of( + "cassandra/schema-keyspace.cql", + "cassandra/schema-ts.cql", + "cassandra/schema-ts-latest.cql" + ); + + @ClassRule(order = 0) + public static final CassandraContainer cassandra = (CassandraContainer) new CassandraContainer("cassandra:4.1") { + @Override + protected void containerIsStarted(InspectContainerResponse containerInfo) { + super.containerIsStarted(containerInfo); + DatabaseDelegate db = new CassandraDatabaseDelegate(this); + INIT_SCRIPTS.forEach(script -> runInitScriptIfRequired(db, script)); + } + + private void runInitScriptIfRequired(DatabaseDelegate db, String initScriptPath) { + logger().info("Init script [{}]", initScriptPath); + if (initScriptPath != null) { + try { + URL resource = Thread.currentThread().getContextClassLoader().getResource(initScriptPath); + if (resource == null) { + logger().warn("Could not load classpath init script: {}", initScriptPath); + throw new ScriptUtils.ScriptLoadException("Could not load classpath init script: " + initScriptPath + ". Resource not found."); + } + String cql = IOUtils.toString(resource, StandardCharsets.UTF_8); + ScriptUtils.executeDatabaseScript(db, initScriptPath, cql); + } catch (IOException e) { + logger().warn("Could not load classpath init script: {}", initScriptPath); + throw new ScriptUtils.ScriptLoadException("Could not load classpath init script: " + initScriptPath, e); + } catch (ScriptException e) { + logger().error("Error while executing init script: {}", initScriptPath, e); + throw new ScriptUtils.UncategorizedScriptException("Error while executing init script: " + initScriptPath, e); + } + } + } + } + .withEnv("HEAP_NEWSIZE", "64M") + .withEnv("MAX_HEAP_SIZE", "512M") + .withEnv("CASSANDRA_CLUSTER_NAME", "ThingsBoard Cluster"); + + @ClassRule(order = 1) + public static ExternalResource resource = new ExternalResource() { + @Override + protected void before() throws Throwable { + cassandra.start(); + String cassandraUrl = String.format("%s:%s", cassandra.getHost(), cassandra.getMappedPort(9042)); + log.debug("Cassandra url [{}]", cassandraUrl); + System.setProperty("cassandra.url", cassandraUrl); + } + + @Override + protected void after() { + cassandra.stop(); + List.of("cassandra.url") + .forEach(System.getProperties()::remove); + } + }; + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisContainer.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisContainer.java new file mode 100644 index 0000000000..e9a6cb8641 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisContainer.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import lombok.extern.slf4j.Slf4j; +import org.junit.ClassRule; +import org.junit.rules.ExternalResource; +import org.testcontainers.containers.GenericContainer; + +import java.util.List; + +@Slf4j +public class AbstractRedisContainer { + + @ClassRule(order = 0) + public static GenericContainer redis = new GenericContainer("redis:7.0") + .withExposedPorts(6379); + + @ClassRule(order = 1) + public static ExternalResource resource = new ExternalResource() { + @Override + protected void before() throws Throwable { + redis.start(); + System.setProperty("cache.type", "redis"); + System.setProperty("redis.connection.type", "standalone"); + System.setProperty("redis.standalone.host", redis.getHost()); + System.setProperty("redis.standalone.port", String.valueOf(redis.getMappedPort(6379))); + } + + @Override + protected void after() { + redis.stop(); + List.of("cache.type", "redis.connection.type", "redis.standalone.host", "redis.standalone.port") + .forEach(System.getProperties()::remove); + } + }; + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/CustomCassandraCQLUnit.java b/dao/src/test/java/org/thingsboard/server/dao/CustomCassandraCQLUnit.java deleted file mode 100644 index 7bab5e1138..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/CustomCassandraCQLUnit.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import com.datastax.oss.driver.api.core.CqlSession; -import org.cassandraunit.BaseCassandraUnit; -import org.cassandraunit.CQLDataLoader; -import org.cassandraunit.dataset.CQLDataSet; -import org.cassandraunit.utils.EmbeddedCassandraServerHelper; - -import java.util.List; - -public class CustomCassandraCQLUnit extends BaseCassandraUnit { - protected List dataSets; - - public CqlSession session; - - public CustomCassandraCQLUnit(List dataSets) { - this.dataSets = dataSets; - } - - public CustomCassandraCQLUnit(List dataSets, int readTimeoutMillis) { - this.dataSets = dataSets; - this.readTimeoutMillis = readTimeoutMillis; - } - - public CustomCassandraCQLUnit(List dataSets, String configurationFileName) { - this(dataSets); - this.configurationFileName = configurationFileName; - } - - public CustomCassandraCQLUnit(List dataSets, String configurationFileName, int readTimeoutMillis) { - this(dataSets); - this.configurationFileName = configurationFileName; - this.readTimeoutMillis = readTimeoutMillis; - } - - public CustomCassandraCQLUnit(List dataSets, String configurationFileName, long startUpTimeoutMillis) { - super(startUpTimeoutMillis); - this.dataSets = dataSets; - this.configurationFileName = configurationFileName; - } - - public CustomCassandraCQLUnit(List dataSets, String configurationFileName, long startUpTimeoutMillis, int readTimeoutMillis) { - super(startUpTimeoutMillis); - this.dataSets = dataSets; - this.configurationFileName = configurationFileName; - this.readTimeoutMillis = readTimeoutMillis; - } - - @Override - protected void load() { - session = EmbeddedCassandraServerHelper.getSession(); - CQLDataLoader dataLoader = new CQLDataLoader(session); - dataSets.forEach(dataLoader::load); - session = dataLoader.getSession(); - System.setSecurityManager(null); - } - - @Override - protected void after() { - super.after(); - try (CqlSession s = session) { - session = null; - } - System.setSecurityManager(null); - } - - // Getters for those who do not like to directly access fields - - public CqlSession getSession() { - return session; - } - -} diff --git a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java index 1cc88f5734..9f82b8735a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java @@ -15,28 +15,14 @@ */ package org.thingsboard.server.dao; -import org.cassandraunit.dataset.cql.ClassPathCQLDataSet; -import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; import org.junit.runner.RunWith; -import java.util.Arrays; - @RunWith(ClasspathSuite.class) @ClassnameFilters({ "org.thingsboard.server.dao.service.*.nosql.*ServiceNoSqlTest", }) -public class NoSqlDaoServiceTestSuite { - - @ClassRule - public static CustomCassandraCQLUnit cassandraUnit = - new CustomCassandraCQLUnit( - Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-keyspace.cql", false, false), - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), - new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) - ), - "cassandra-test.yaml", 30000L); +public class NoSqlDaoServiceTestSuite extends AbstractNoSqlContainer { } diff --git a/dao/src/test/java/org/thingsboard/server/dao/RedisSqlTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/RedisSqlTestSuite.java index 2fa63f2387..61f77ee1b7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/RedisSqlTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/RedisSqlTestSuite.java @@ -15,37 +15,15 @@ */ package org.thingsboard.server.dao; -import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; import org.junit.runner.RunWith; -import org.springframework.context.ApplicationContextInitializer; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.support.TestPropertySourceUtils; -import org.testcontainers.containers.GenericContainer; -@ContextConfiguration(initializers = RedisSqlTestSuite.class) @RunWith(ClasspathSuite.class) @ClassnameFilters( //All the same tests using redis instead of caffeine. "org.thingsboard.server.dao.service.*ServiceSqlTest" ) -public class RedisSqlTestSuite implements ApplicationContextInitializer { - - @ClassRule - public static GenericContainer redis = new GenericContainer("redis:4.0").withExposedPorts(6379); - - @Override - public void initialize(ConfigurableApplicationContext applicationContext) { - TestPropertySourceUtils.addInlinedPropertiesToEnvironment( - applicationContext, "cache.type=redis"); - TestPropertySourceUtils.addInlinedPropertiesToEnvironment( - applicationContext, "redis.connection.type=standalone"); - TestPropertySourceUtils.addInlinedPropertiesToEnvironment( - applicationContext, "redis.standalone.host=localhost"); - TestPropertySourceUtils.addInlinedPropertiesToEnvironment( - applicationContext, "redis.standalone.port=" + redis.getMappedPort(6379)); - } +public class RedisSqlTestSuite extends AbstractRedisContainer { } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java index 9415da99ea..7fc9f7b218 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java @@ -43,8 +43,8 @@ import static org.mockito.Mockito.when; public abstract class BaseDeviceCredentialsCacheTest extends AbstractServiceTest { - private static final String CREDENTIALS_ID_1 = StringUtils.randomAlphanumeric(20); - private static final String CREDENTIALS_ID_2 = StringUtils.randomAlphanumeric(20); + private final String CREDENTIALS_ID_1 = StringUtils.randomAlphanumeric(20); + private final String CREDENTIALS_ID_2 = StringUtils.randomAlphanumeric(20); @Autowired private DeviceCredentialsService deviceCredentialsService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 2054945285..07b17182a5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -26,7 +26,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -50,6 +49,7 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.AssetTypeFilter; import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; @@ -62,6 +62,7 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; @@ -106,9 +107,6 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { private TenantId tenantId; - @Autowired - private JdbcTemplate template; - @Autowired private RelationRepository relationRepository; @@ -986,6 +984,360 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { assertEquals(devices.size(), result.getTotalElements()); } + @Test + public void testFindEntityDataByQuery_filter_entity_name_starts_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("Device"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%Device"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_entity_name_ends_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_entity_name_contains() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device test" + i); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_starts_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("Device"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%Device"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_ends_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_contains() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device test" + i); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_starts_with() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset " + i + " test"); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("Asset"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("Asset%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%Asset%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%Asset"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_ends_with() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset " + i + " test"); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_contains() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset test" + i); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + private PageData searchEntities(EntityDataQuery query) { return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); } diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 15131011d8..a2a21abc11 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -7,10 +7,9 @@ updates.enabled=false audit-log.enabled=true audit-log.sink.type=none -cache.type=caffeine +#cache.type=caffeine # will be injected redis by RedisContainer or will be default (caffeine) cache.maximumPoolSize=16 cache.attributes.enabled=true -#cache.type=redis cache.specs.relations.timeToLiveInMinutes=1440 cache.specs.relations.maxSize=100000 diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index 5765153a6f..5876582ee6 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -2,7 +2,7 @@ cassandra.cluster_name=Thingsboard Cluster cassandra.keyspace_name=thingsboard -cassandra.url=127.0.0.1:9142 +#cassandra.url=127.0.0.1:9142 # will be injected by NoSqlContainer cassandra.local_datacenter=datacenter1 diff --git a/dao/src/test/resources/logback.xml b/dao/src/test/resources/logback.xml index f74ba574b5..61397ec6f1 100644 --- a/dao/src/test/resources/logback.xml +++ b/dao/src/test/resources/logback.xml @@ -8,9 +8,7 @@ - - - + diff --git a/dao/src/test/resources/nosql-test.properties b/dao/src/test/resources/nosql-test.properties index 1c6d2442cf..d63a9f7f82 100644 --- a/dao/src/test/resources/nosql-test.properties +++ b/dao/src/test/resources/nosql-test.properties @@ -15,7 +15,7 @@ spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver -spring.datasource.hikari.maximumPoolSize = 50 +spring.datasource.hikari.maximumPoolSize=16 queue.rule-engine.queues[0].name=Main queue.rule-engine.queues[0].topic=tb_rule_engine.main diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index b2ded716c4..2fed33b43a 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,7 +16,7 @@ spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver -spring.datasource.hikari.maximumPoolSize = 50 +spring.datasource.hikari.maximumPoolSize=16 service.type=monolith diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index ae0e8b79a0..5b63ba712a 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -18,7 +18,7 @@ As result, in REPOSITORY column, next images should be present: thingsboard/tb-web-ui thingsboard/tb-js-executor -- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone: +- Run the black box tests (without ui tests) in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone: mvn clean install -DblackBoxTests.skip=false @@ -34,11 +34,23 @@ As result, in REPOSITORY column, next images should be present: mvn clean install -DblackBoxTests.skip=false -DrunLocal=true -- To run ui smoke tests in the [msa/black-box-tests](../black-box-tests) directory specifying suite name: +- To run only ui tests in the [msa/black-box-tests](../black-box-tests) directory: mvn clean install -DblackBoxTests.skip=false -Dsuite=uiTests -- To run all tests in the [msa/black-box-tests](../black-box-tests) directory specifying suite name: +- To run only ui smoke rule chains tests in the [msa/black-box-tests](../black-box-tests) directory: + + mvn clean install -DblackBoxTests.skip=false -Dsuite=smokesRuleChain + +- To run only ui smoke customers tests in the [msa/black-box-tests](../black-box-tests) directory: + + mvn clean install -DblackBoxTests.skip=false -Dsuite=smokesCustomer + +- To run only ui smoke profiles tests in the [msa/black-box-tests](../black-box-tests) directory: + + mvn clean install -DblackBoxTests.skip=false -Dsuite=smokesPrifiles + +- To run all tests (black-box and ui) in the [msa/black-box-tests](../black-box-tests) directory: mvn clean install -DblackBoxTests.skip=false -Dsuite=all diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 95e2877904..ffbe14ed52 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -106,6 +106,7 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), + new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java index 32d5b0f9d7..be9a5c422a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -16,6 +16,7 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.DockerClientFactory; import java.io.IOException; import java.io.InputStream; @@ -41,7 +42,9 @@ public class TestProperties { public static String getBaseUiUrl() { if (instance.isActive()) { - return "https://host.docker.internal"; + //return "https://host.docker.internal" // this alternative requires docker-selenium.yml extra_hosts: - "host.docker.internal:host-gateway" + //return "https://" + DockerClientFactory.instance().dockerHostIpAddress(); //this alternative will get Docker IP from testcontainers + return "https://haproxy"; //communicate inside current docker-compose network to the load balancer container } return getProperties().getProperty("tb.baseUiUrl"); } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 4b07a72772..e6c9856094 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -82,6 +82,9 @@ public class ThingsBoardDbInstaller { )); if (IS_HYBRID_MODE) { composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); + composeFiles.add(new File("src/test/resources/docker-compose.hybrid-test-extras.yml")); + } else { + composeFiles.add(new File("src/test/resources/docker-compose.postgres-test-extras.yml")); } String identifier = Base58.randomString(6).toLowerCase(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java index f8631bd198..cd4ca51698 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java @@ -32,9 +32,11 @@ import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; @Slf4j abstract public class AbstractBasePage { + public static final long WAIT_TIMEOUT = TimeUnit.SECONDS.toMillis(30); protected WebDriver driver; protected WebDriverWait wait; protected Actions actions; @@ -43,7 +45,7 @@ abstract public class AbstractBasePage { public AbstractBasePage(WebDriver driver) { this.driver = driver; - this.wait = new WebDriverWait(driver, Duration.ofMillis(8000)); + this.wait = new WebDriverWait(driver, Duration.ofMillis(WAIT_TIMEOUT)); this.actions = new Actions(driver); this.js = (JavascriptExecutor) driver; } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/pages/OtherPageElements.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/pages/OtherPageElements.java index 3b18df801f..99e737dba1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/pages/OtherPageElements.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/pages/OtherPageElements.java @@ -42,7 +42,7 @@ public class OtherPageElements extends AbstractBasePage { private static final String MARKS_CHECKBOX = "//mat-row[contains (@class,'mat-selected')]//mat-checkbox[contains(@class, 'checked')]"; private static final String SELECT_ALL_CHECKBOX = "//thead//mat-checkbox"; private static final String ALL_ENTITY = "//mat-row[@class='mat-row cdk-row mat-row-select ng-star-inserted']"; - private static final String EDIT_PENCIL_BTN = "//mat-icon[contains(text(),'edit')]/ancestor::button"; + private static final String EDIT_PENCIL_BTN = "//tb-details-panel//mat-icon[contains(text(),'edit')]/ancestor::button"; private static final String NAME_FIELD_EDIT_VIEW = "//input[@formcontrolname='name']"; private static final String HEADER_NAME_VIEW = "//header//div[@class='tb-details-title']/span"; private static final String DONE_BTN_EDIT_VIEW = "//mat-icon[contains(text(),'done')]/ancestor::button"; diff --git a/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml new file mode 100644 index 0000000000..5df6d2582d --- /dev/null +++ b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + cassandra: + environment: + HEAP_NEWSIZE: 128M + MAX_HEAP_SIZE: 1024M diff --git a/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml new file mode 100644 index 0000000000..90d7a351cf --- /dev/null +++ b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml @@ -0,0 +1,19 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +# Placeholder diff --git a/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml b/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml index 414aef5568..f1c2f0b41c 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml @@ -14,7 +14,7 @@ # limitations under the License. # -version: '2.2' +version: '3.0' services: rabbitmq: diff --git a/msa/black-box-tests/src/test/resources/docker-selenium.yml b/msa/black-box-tests/src/test/resources/docker-selenium.yml index 38a3b74bfe..2a88264bc8 100644 --- a/msa/black-box-tests/src/test/resources/docker-selenium.yml +++ b/msa/black-box-tests/src/test/resources/docker-selenium.yml @@ -31,7 +31,6 @@ services: SE_SCREEN_HEIGHT: 1080 SE_SCREEN_DEPTH: 24 SE_SCREEN_DPI: 74 - extra_hosts: - - "host.docker.internal:172.17.0.1" - - +# Alternative way how to connect to the host address +# extra_hosts: +# - "host.docker.internal:host-gateway" diff --git a/msa/pom.xml b/msa/pom.xml index 515557b018..ad40e574ec 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -39,13 +39,14 @@ + tb + web-ui vc-executor vc-executor-docker - js-executor - web-ui tb-node transport + js-executor diff --git a/pom.xml b/pom.xml index 61053bebc7..77aeb81498 100755 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ 2.0.51.Final 1.7.0 4.8.0 - 3.0.0-M6 + 3.0.0-M9 3.0.2 3.0.4 1.6.3 @@ -126,7 +126,6 @@ 2.8.5 4.1.0 - 4.3.1.0 2.7.2 1.5.2 5.8.2 @@ -659,7 +658,7 @@ ${surefire.version} - --illegal-access=permit + --illegal-access=permit -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=20 @@ -1603,26 +1602,6 @@ - - org.cassandraunit - cassandra-unit - ${cassandra-unit.version} - test - - - junit - junit - - - org.hamcrest - hamcrest-core - - - org.hamcrest - hamcrest-library - - - org.apache.cassandra cassandra-all @@ -1746,6 +1725,12 @@ bcpkix-jdk15on ${bouncycastle.version} + + org.testcontainers + cassandra + ${testcontainers.version} + test + org.testcontainers postgresql diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 56621162f7..e85f0094b5 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -150,22 +150,6 @@ mockserver-client-java test - - - org.cassandraunit - cassandra-unit - - - org.slf4j - slf4j-log4j12 - - - org.hibernate - hibernate-validator - - - test - com.jayway.jsonpath json-path diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 48d71873eb..0137887aaf 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -57,6 +57,9 @@ import java.security.cert.X509Certificate; import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.RSAPrivateCrtKeySpec; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -104,7 +107,7 @@ public class CertPemCredentials implements ClientCredentials { } private KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { - X509Certificate certHolder = readCertFile(cert); + List certHolders = readCertFile(cert); Object keyObject = readPrivateKeyFile(privateKey); char[] passwordCharArray = "".toCharArray(); if (!StringUtils.isEmpty(password)) { @@ -129,43 +132,53 @@ public class CertPemCredentials implements ClientCredentials { KeyStore clientKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); clientKeyStore.load(null, null); - clientKeyStore.setCertificateEntry("cert", certHolder); + for (X509Certificate certHolder : certHolders) { + clientKeyStore.setCertificateEntry("cert-" + certHolder.getSubjectDN().getName(), certHolder); + } clientKeyStore.setKeyEntry("private-key", privateKey, passwordCharArray, - new Certificate[]{certHolder}); - + certHolders.toArray(new Certificate[]{})); KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keyManagerFactory.init(clientKeyStore, passwordCharArray); return keyManagerFactory; } protected TrustManagerFactory createAndInitTrustManagerFactory() throws Exception { - X509Certificate caCertHolder; - caCertHolder = readCertFile(caCert); + List caCertHolders = readCertFile(caCert); KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); - caKeyStore.setCertificateEntry("caCert-cert", caCertHolder); + for (X509Certificate caCertHolder : caCertHolders) { + caKeyStore.setCertificateEntry("caCert-cert-" + caCertHolder.getSubjectDN().getName(), caCertHolder); + } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); trustManagerFactory.init(caKeyStore); return trustManagerFactory; } - private X509Certificate readCertFile(String fileContent) throws Exception { - X509Certificate certificate = null; - if (fileContent != null && !fileContent.trim().isEmpty()) { - fileContent = fileContent.replace("-----BEGIN CERTIFICATE-----", "") + List readCertFile(String fileContent) throws Exception { + if (fileContent == null || fileContent.trim().isEmpty()) { + return Collections.emptyList(); + } + + List certificates = new ArrayList<>(); + String[] pems = fileContent.trim().split("-----END CERTIFICATE-----"); + for (String pem : pems) { + if (pem.trim().isEmpty()) { + continue; + } + pem = pem.replace("-----BEGIN CERTIFICATE-----", "") .replace("-----END CERTIFICATE-----", "") .replaceAll("\\s", ""); - byte[] decoded = Base64.decodeBase64(fileContent); + byte[] decoded = Base64.decodeBase64(pem); CertificateFactory certFactory = CertificateFactory.getInstance("X.509"); try (InputStream inStream = new ByteArrayInputStream(decoded)) { - certificate = (X509Certificate) certFactory.generateCertificate(inStream); + certificates.add((X509Certificate) certFactory.generateCertificate(inStream)); } } - return certificate; + return certificates; } private PrivateKey readPrivateKeyFile(String fileContent) throws Exception { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java new file mode 100644 index 0000000000..904f470001 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.credentials; + +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.security.cert.X509Certificate; +import java.util.List; + +public class CertPemCredentialsTest { + + private final CertPemCredentials credentials = new CertPemCredentials(); + + @Test + public void testChainOfCertificates() throws Exception { + String fileContent = fileContent("pem/tb-cloud-chain.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(4, x509Certificates.size()); + Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + x509Certificates.get(0).getSubjectDN().getName()); + Assert.assertEquals("CN=Sectigo ECC Organization Validation Secure Server CA, O=Sectigo Limited, L=Salford, ST=Greater Manchester, C=GB", + x509Certificates.get(1).getSubjectDN().getName()); + Assert.assertEquals("CN=USERTrust ECC Certification Authority, O=The USERTRUST Network, L=Jersey City, ST=New Jersey, C=US", + x509Certificates.get(2).getSubjectDN().getName()); + Assert.assertEquals("CN=AAA Certificate Services, O=Comodo CA Limited, L=Salford, ST=Greater Manchester, C=GB", + x509Certificates.get(3).getSubjectDN().getName()); + } + + @Test + public void testSingleCertificate() throws Exception { + String fileContent = fileContent("pem/tb-cloud.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(1, x509Certificates.size()); + Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + x509Certificates.get(0).getSubjectDN().getName()); + } + + @Test + public void testEmptyFileContent() throws Exception { + String fileContent = fileContent("pem/empty.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(0, x509Certificates.size()); + } + + private String fileContent(String fileName) throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + File file = new File(classLoader.getResource(fileName).getFile()); + return FileUtils.readFileToString(file, "UTF-8"); + } +} diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/empty.pem b/rule-engine/rule-engine-components/src/test/resources/pem/empty.pem new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem new file mode 100644 index 0000000000..23049273c7 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem @@ -0,0 +1,103 @@ +-----BEGIN CERTIFICATE----- +MIIFejCCBSCgAwIBAgIQT2YV5NVp2PAY1O5rxMlj6DAKBggqhkjOPQQDAjCBlTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRT +ZWN0aWdvIEVDQyBPcmdhbml6YXRpb24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVy +IENBMB4XDTIxMTAwNTAwMDAwMFoXDTIyMTAwNTIzNTk1OVowWjELMAkGA1UEBhMC +VVMxETAPBgNVBAgTCE5ldyBZb3JrMRowGAYDVQQKExFUaGluZ3NCb2FyZCwgSW5j +LjEcMBoGA1UEAwwTKi50aGluZ3Nib2FyZC5jbG91ZDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABNkK/UerQZXPP0H/Tl8YhRPlzW85yTAcQ5hXhs2fyXn7Bdj4EueQuZrv +Pw98xwHJr87jslFbS/WiSdtBYPvjsUyXqh7aMvOcEhSgEOWDmtoj3P1Xk1hNLb6m +xAQfFL8cZ6OCA20wggNpMB8GA1UdIwQYMBaAFE1K78RGsxKtT06asVniUasIEHgI +MB0GA1UdDgQWBBRr4HG23dsao68r9r7obGxB70ptBzAOBgNVHQ8BAf8EBAMCB4Aw +DAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwSgYD +VR0gBEMwQTA1BgwrBgEEAbIxAQIBAwQwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9z +ZWN0aWdvLmNvbS9DUFMwCAYGZ4EMAQICMFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6 +Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb0VDQ09yZ2FuaXphdGlvblZhbGlkYXRp +b25TZWN1cmVTZXJ2ZXJDQS5jcmwwgYoGCCsGAQUFBwEBBH4wfDBVBggrBgEFBQcw +AoZJaHR0cDovL2NydC5zZWN0aWdvLmNvbS9TZWN0aWdvRUNDT3JnYW5pemF0aW9u +VmFsaWRhdGlvblNlY3VyZVNlcnZlckNBLmNydDAjBggrBgEFBQcwAYYXaHR0cDov +L29jc3Auc2VjdGlnby5jb20wMQYDVR0RBCowKIITKi50aGluZ3Nib2FyZC5jbG91 +ZIIRdGhpbmdzYm9hcmQuY2xvdWQwggGABgorBgEEAdZ5AgQCBIIBcASCAWwBagB2 +AEalVet1+pEgMLWiiWn0830RLEF0vv1JuIWr8vxw/m1HAAABfFDbhtMAAAQDAEcw +RQIhAKNykhkQTngK0yOcYHGHUQSy6JmJYl+5nc1qELirPHwAAiBgV2Db5ZFHNvzn +zp9Ob/OG0o36z6rcilbLI/daZwnyewB3AEHIyrHfIkZKEMahOglCh15OMYsbA+vr +S8do8JBilgb2AAABfFDbhqYAAAQDAEgwRgIhALFvTbapKhO7DPrF6KtE9sjFMMth +qjqaeaHYN6JGnUAIAiEApEW+rxlzxH1+qEwJrFyQLr5rSKTuEoSjv3hbrzb9GQ4A +dwApeb7wnjk5IfBWc59jpXflvld9nGAK+PlNXSZcJV3HhAAAAXxQ24ZnAAAEAwBI +MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL +1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ +GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR +13sr0/ED+2PUom2dnfg= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDrjCCAzOgAwIBAgIQNb50Y4yz6d4oBXC3l4CzZzAKBggqhkjOPQQDAzCBiDEL +MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl +eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT +JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTgxMTAy +MDAwMDAwWhcNMzAxMjMxMjM1OTU5WjCBlTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEYMBYGA1UEChMP +U2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRTZWN0aWdvIEVDQyBPcmdhbml6YXRp +b24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVyIENBMFkwEwYHKoZIzj0CAQYIKoZI +zj0DAQcDQgAEnI5cCmFvoVij0NXO+vxE+f+6Bh57FhpyH0LTCrJmzfsPSXIhTSex +r92HOlz+aHqoGE0vSe/CSwLFoWcZ8W1jOaOCAW4wggFqMB8GA1UdIwQYMBaAFDrh +CYbUzxnClnZ0SXbc4DXGY2OaMB0GA1UdDgQWBBRNSu/ERrMSrU9OmrFZ4lGrCBB4 +CDAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwGwYDVR0gBBQwEjAGBgRVHSAAMAgGBmeBDAEC +AjBQBgNVHR8ESTBHMEWgQ6BBhj9odHRwOi8vY3JsLnVzZXJ0cnVzdC5jb20vVVNF +UlRydXN0RUNDQ2VydGlmaWNhdGlvbkF1dGhvcml0eS5jcmwwdgYIKwYBBQUHAQEE +ajBoMD8GCCsGAQUFBzAChjNodHRwOi8vY3J0LnVzZXJ0cnVzdC5jb20vVVNFUlRy +dXN0RUNDQWRkVHJ1c3RDQS5jcnQwJQYIKwYBBQUHMAGGGWh0dHA6Ly9vY3NwLnVz +ZXJ0cnVzdC5jb20wCgYIKoZIzj0EAwMDaQAwZgIxAOk//uo7i/MoeKdcyeqvjOXs +BJFGLI+1i0d+Tty7zEnn2w4DNS21TK8wmY3Kjm3EmQIxAPI1qHM/I+OS+hx0OZhG +fDoNifTe/GxgWZ1gOYQKzn6lwP0yGKlrP+7vrVC8IczJ4A== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIID0zCCArugAwIBAgIQVmcdBOpPmUxvEIFHWdJ1lDANBgkqhkiG9w0BAQwFADB7 +MQswCQYDVQQGEwJHQjEbMBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYD +VQQHDAdTYWxmb3JkMRowGAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UE +AwwYQUFBIENlcnRpZmljYXRlIFNlcnZpY2VzMB4XDTE5MDMxMjAwMDAwMFoXDTI4 +MTIzMTIzNTk1OVowgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpOZXcgSmVyc2V5 +MRQwEgYDVQQHEwtKZXJzZXkgQ2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBO +ZXR3b3JrMS4wLAYDVQQDEyVVU0VSVHJ1c3QgRUNDIENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEGqxUWqn5aCPnetUkb1PGWthL +q8bVttHmc3Gu3ZzWDGH926CJA7gFFOxXzu5dP+Ihs8731Ip54KODfi2X0GHE8Znc +JZFjq38wo7Rw4sehM5zzvy5cU7Ffs30yf4o043l5o4HyMIHvMB8GA1UdIwQYMBaA +FKARCiM+lvEH7OKvKe+CpX/QMKS0MB0GA1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1 +xmNjmjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zARBgNVHSAECjAI +MAYGBFUdIAAwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5j +b20vQUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNAYIKwYBBQUHAQEEKDAmMCQG +CCsGAQUFBzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQEM +BQADggEBABns652JLCALBIAdGN5CmXKZFjK9Dpx1WywV4ilAbe7/ctvbq5AfjJXy +ij0IckKJUAfiORVsAYfZFhr1wHUrxeZWEQff2Ji8fJ8ZOd+LygBkc7xGEJuTI42+ +FsMuCIKchjN0djsoTI0DQoWz4rIjQtUfenVqGtF8qmchxDM6OW1TyaLtYiKou+JV +bJlsQ2uRl9EMC5MCHdK8aXdJ5htN978UeAOwproLtOGFfy/cQjutdAFI3tZs4RmY +CV4Ks2dH/hzg1cEo70qLRDEmBDeNiXQ2Lu+lIg+DdEmSx/cQwgwp+7e9un/jX9Wf +8qn0dNW44bOwgeThpWOjzOoEeJBuv/c= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- + diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem new file mode 100644 index 0000000000..8aba29c995 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFejCCBSCgAwIBAgIQT2YV5NVp2PAY1O5rxMlj6DAKBggqhkjOPQQDAjCBlTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRT +ZWN0aWdvIEVDQyBPcmdhbml6YXRpb24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVy +IENBMB4XDTIxMTAwNTAwMDAwMFoXDTIyMTAwNTIzNTk1OVowWjELMAkGA1UEBhMC +VVMxETAPBgNVBAgTCE5ldyBZb3JrMRowGAYDVQQKExFUaGluZ3NCb2FyZCwgSW5j +LjEcMBoGA1UEAwwTKi50aGluZ3Nib2FyZC5jbG91ZDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABNkK/UerQZXPP0H/Tl8YhRPlzW85yTAcQ5hXhs2fyXn7Bdj4EueQuZrv +Pw98xwHJr87jslFbS/WiSdtBYPvjsUyXqh7aMvOcEhSgEOWDmtoj3P1Xk1hNLb6m +xAQfFL8cZ6OCA20wggNpMB8GA1UdIwQYMBaAFE1K78RGsxKtT06asVniUasIEHgI +MB0GA1UdDgQWBBRr4HG23dsao68r9r7obGxB70ptBzAOBgNVHQ8BAf8EBAMCB4Aw +DAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwSgYD +VR0gBEMwQTA1BgwrBgEEAbIxAQIBAwQwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9z +ZWN0aWdvLmNvbS9DUFMwCAYGZ4EMAQICMFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6 +Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb0VDQ09yZ2FuaXphdGlvblZhbGlkYXRp +b25TZWN1cmVTZXJ2ZXJDQS5jcmwwgYoGCCsGAQUFBwEBBH4wfDBVBggrBgEFBQcw +AoZJaHR0cDovL2NydC5zZWN0aWdvLmNvbS9TZWN0aWdvRUNDT3JnYW5pemF0aW9u +VmFsaWRhdGlvblNlY3VyZVNlcnZlckNBLmNydDAjBggrBgEFBQcwAYYXaHR0cDov +L29jc3Auc2VjdGlnby5jb20wMQYDVR0RBCowKIITKi50aGluZ3Nib2FyZC5jbG91 +ZIIRdGhpbmdzYm9hcmQuY2xvdWQwggGABgorBgEEAdZ5AgQCBIIBcASCAWwBagB2 +AEalVet1+pEgMLWiiWn0830RLEF0vv1JuIWr8vxw/m1HAAABfFDbhtMAAAQDAEcw +RQIhAKNykhkQTngK0yOcYHGHUQSy6JmJYl+5nc1qELirPHwAAiBgV2Db5ZFHNvzn +zp9Ob/OG0o36z6rcilbLI/daZwnyewB3AEHIyrHfIkZKEMahOglCh15OMYsbA+vr +S8do8JBilgb2AAABfFDbhqYAAAQDAEgwRgIhALFvTbapKhO7DPrF6KtE9sjFMMth +qjqaeaHYN6JGnUAIAiEApEW+rxlzxH1+qEwJrFyQLr5rSKTuEoSjv3hbrzb9GQ4A +dwApeb7wnjk5IfBWc59jpXflvld9nGAK+PlNXSZcJV3HhAAAAXxQ24ZnAAAEAwBI +MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL +1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ +GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR +13sr0/ED+2PUom2dnfg= +-----END CERTIFICATE-----