Browse Source

Merge remote-tracking branch 'upstream/master'

pull/1243/head
mp-loki 8 years ago
parent
commit
5f16cbe23c
  1. 46
      application/src/main/data/json/system/widget_bundles/input_widgets.json
  2. 2
      application/src/main/data/upgrade/2.2.0/schema_update.sql
  3. 5
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  4. 5
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  5. 2
      application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java
  6. 90
      application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java
  7. 5
      application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java
  8. 1
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  9. 156
      application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java
  10. 15
      application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java
  11. 147
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java
  12. 7
      application/src/main/java/org/thingsboard/server/service/telemetry/TelemetryWebSocketMsgEndpoint.java
  13. 1
      application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java
  14. 6
      application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java
  15. 23
      application/src/main/resources/thingsboard.yml
  16. 4
      common/data/src/main/java/org/thingsboard/server/common/data/exception/ThingsboardErrorCode.java
  17. 8
      common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimitsException.java
  18. 33
      common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java
  19. 32
      common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java
  20. 4
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java
  21. 12
      dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java
  22. 8
      dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java
  23. 3
      dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java
  24. 11
      dao/src/main/java/org/thingsboard/server/dao/service/TimePaginatedRemover.java
  25. 7
      dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java
  26. 2
      dao/src/main/resources/sql/schema-entities.sql
  27. 7
      docker/docker-compose.yml
  28. 2
      docker/kafka.env
  29. 2
      docker/tb-node.env
  30. 43
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java
  31. 4
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java
  32. 2
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java
  33. 101
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java
  34. 34
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNodeConfiguration.java
  35. 6
      rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js
  36. 7
      transport/coap/src/main/resources/tb-coap-transport.yml
  37. 5
      transport/http/src/main/resources/tb-http-transport.yml
  38. 5
      transport/mqtt/src/main/resources/tb-mqtt-transport.yml
  39. 29
      ui/src/app/api/telemetry-websocket.service.js
  40. 4
      ui/src/app/common/types.constant.js
  41. 3
      ui/src/app/services/toast.scss

46
application/src/main/data/json/system/widget_bundles/input_widgets.json

File diff suppressed because one or more lines are too long

2
application/src/main/data/upgrade/2.2.0/schema_update.sql

@ -15,3 +15,5 @@
--
ALTER TABLE component_descriptor ADD UNIQUE (clazz);
ALTER TABLE entity_view ALTER COLUMN keys SET DATA TYPE varchar(10000000);

5
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -56,6 +56,7 @@ import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.kafka.TbNodeIdProvider;
import org.thingsboard.server.service.cluster.discovery.DiscoveryService;
import org.thingsboard.server.service.cluster.routing.ClusterRoutingService;
import org.thingsboard.server.service.cluster.rpc.ClusterRpcService;
@ -276,6 +277,10 @@ public class ActorSystemContext {
@Setter
private ActorSystem actorSystem;
@Autowired
@Getter
private TbNodeIdProvider nodeIdProvider;
@Getter
@Setter
private ActorRef appActor;

5
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -161,6 +161,11 @@ class DefaultTbContext implements TbContext {
return new RuleNodeJsScriptEngine(mainCtx.getJsSandbox(), nodeCtx.getSelf().getId(), script, argNames);
}
@Override
public String getNodeId() {
return mainCtx.getNodeIdProvider().getNodeId();
}
@Override
public AttributesService getAttributesService() {
return mainCtx.getAttributesService();

2
application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java

@ -99,7 +99,7 @@ public class DefaultActorService implements ActorService {
@PostConstruct
public void initActorSystem() {
log.info("Initializing Actor system. {}", actorContext.getRuleChainService());
log.info("Initializing Actor system.");
actorContext.setActorService(this);
system = ActorSystem.create(ACTOR_SYSTEM_NAME, actorContext.getConfig());
actorContext.setActorSystem(system);

90
application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java

@ -0,0 +1,90 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.GenericFilterBean;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.exception.ThingsboardErrorResponseHandler;
import org.thingsboard.server.service.security.model.SecurityUser;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@Component
public class RateLimitProcessingFilter extends GenericFilterBean {
@Value("${server.rest.limits.tenant.enabled:false}")
private boolean perTenantLimitsEnabled;
@Value("${server.rest.limits.tenant.configuration:}")
private String perTenantLimitsConfiguration;
@Value("${server.rest.limits.customer.enabled:false}")
private boolean perCustomerLimitsEnabled;
@Value("${server.rest.limits.customer.configuration:}")
private String perCustomerLimitsConfiguration;
@Autowired
private ThingsboardErrorResponseHandler errorResponseHandler;
private ConcurrentMap<TenantId, TbRateLimits> perTenantLimits = new ConcurrentHashMap<>();
private ConcurrentMap<CustomerId, TbRateLimits> perCustomerLimits = new ConcurrentHashMap<>();
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
SecurityUser user = getCurrentUser();
if (user != null && !user.isSystemAdmin()) {
if (perTenantLimitsEnabled) {
TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(user.getTenantId(), id -> new TbRateLimits(perTenantLimitsConfiguration));
if (!rateLimits.tryConsume()) {
errorResponseHandler.handle(new TbRateLimitsException(EntityType.TENANT), (HttpServletResponse) response);
return;
}
}
if (perCustomerLimitsEnabled && user.isCustomerUser()) {
TbRateLimits rateLimits = perCustomerLimits.computeIfAbsent(user.getCustomerId(), id -> new TbRateLimits(perCustomerLimitsConfiguration));
if (!rateLimits.tryConsume()) {
errorResponseHandler.handle(new TbRateLimitsException(EntityType.CUSTOMER), (HttpServletResponse) response);
return;
}
}
}
chain.doFilter(request, response);
}
protected SecurityUser getCurrentUser() {
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
if (authentication != null && authentication.getPrincipal() instanceof SecurityUser) {
return (SecurityUser) authentication.getPrincipal();
} else {
return null;
}
}
}

5
application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java

@ -91,6 +91,8 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt
@Autowired private ObjectMapper objectMapper;
@Autowired private RateLimitProcessingFilter rateLimitProcessingFilter;
@Bean
protected RestLoginProcessingFilter buildRestLoginProcessingFilter() throws Exception {
RestLoginProcessingFilter filter = new RestLoginProcessingFilter(FORM_BASED_LOGIN_ENTRY_POINT, successHandler, failureHandler, objectMapper);
@ -186,7 +188,8 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt
.addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class);
.addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class);
}

1
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -52,6 +52,7 @@ import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.cluster.SendToClusterMsg;
import org.thingsboard.server.common.msg.system.ServiceToRuleEngineMsg;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.attributes.AttributesService;

156
application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java

@ -18,14 +18,21 @@ package org.thingsboard.server.controller.plugin;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.BeanCreationNotAllowedException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.TextWebSocketHandler;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.config.WebSocketConfiguration;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.telemetry.SessionEvent;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketMsgEndpoint;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService;
@ -34,6 +41,7 @@ import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef;
import java.io.IOException;
import java.net.URI;
import java.security.InvalidParameterException;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@ -48,12 +56,32 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
@Autowired
private TelemetryWebSocketService webSocketService;
@Value("${server.ws.limits.max_sessions_per_tenant:0}")
private int maxSessionsPerTenant;
@Value("${server.ws.limits.max_sessions_per_customer:0}")
private int maxSessionsPerCustomer;
@Value("${server.ws.limits.max_sessions_per_regular_user:0}")
private int maxSessionsPerRegularUser;
@Value("${server.ws.limits.max_sessions_per_public_user:0}")
private int maxSessionsPerPublicUser;
@Value("${server.ws.limits.max_updates_per_session:}")
private String perSessionUpdatesConfiguration;
private ConcurrentMap<String, TelemetryWebSocketSessionRef> blacklistedSessions = new ConcurrentHashMap<>();
private ConcurrentMap<String, TbRateLimits> perSessionUpdateLimits = new ConcurrentHashMap<>();
private ConcurrentMap<TenantId, Set<String>> tenantSessionsMap = new ConcurrentHashMap<>();
private ConcurrentMap<CustomerId, Set<String>> customerSessionsMap = new ConcurrentHashMap<>();
private ConcurrentMap<UserId, Set<String>> regularUserSessionsMap = new ConcurrentHashMap<>();
private ConcurrentMap<UserId, Set<String>> publicUserSessionsMap = new ConcurrentHashMap<>();
@Override
public void handleTextMessage(WebSocketSession session, TextMessage message) {
try {
log.info("[{}] Processing {}", session.getId(), message);
SessionMetaData sessionMd = internalSessionMap.get(session.getId());
if (sessionMd != null) {
log.info("[{}][{}] Processing {}", sessionMd.sessionRef.getSecurityCtx().getTenantId(), session.getId(), message.getPayload());
webSocketService.handleWebSocketMsg(sessionMd.sessionRef, message.getPayload());
} else {
log.warn("[{}] Failed to find session", session.getId());
@ -71,12 +99,15 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
String internalSessionId = session.getId();
TelemetryWebSocketSessionRef sessionRef = toRef(session);
String externalSessionId = sessionRef.getSessionId();
if (!checkLimits(session, sessionRef)) {
return;
}
internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef));
externalSessionMap.put(externalSessionId, internalSessionId);
processInWebSocketService(sessionRef, SessionEvent.onEstablished());
log.info("[{}][{}] Session is started", externalSessionId, session.getId());
log.info("[{}][{}][{}] Session is opened", sessionRef.getSecurityCtx().getTenantId(), externalSessionId, session.getId());
} catch (InvalidParameterException e) {
log.warn("[[{}] Failed to start session", session.getId(), e);
log.warn("[{}] Failed to start session", session.getId(), e);
session.close(CloseStatus.BAD_DATA.withReason(e.getMessage()));
} catch (Exception e) {
log.warn("[{}] Failed to start session", session.getId(), e);
@ -101,6 +132,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
super.afterConnectionClosed(session, closeStatus);
SessionMetaData sessionMd = internalSessionMap.remove(session.getId());
if (sessionMd != null) {
cleanupLimits(session, sessionMd.sessionRef);
externalSessionMap.remove(sessionMd.sessionRef.getSessionId());
processInWebSocketService(sessionMd.sessionRef, SessionEvent.onClosed());
}
@ -136,7 +168,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
private final WebSocketSession session;
private final TelemetryWebSocketSessionRef sessionRef;
public SessionMetaData(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) {
SessionMetaData(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) {
super();
this.session = session;
this.sessionRef = sessionRef;
@ -144,13 +176,29 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
@Override
public void send(TelemetryWebSocketSessionRef sessionRef, String msg) throws IOException {
public void send(TelemetryWebSocketSessionRef sessionRef, int subscriptionId, String msg) throws IOException {
String externalId = sessionRef.getSessionId();
log.debug("[{}] Processing {}", externalId, msg);
String internalId = externalSessionMap.get(externalId);
if (internalId != null) {
SessionMetaData sessionMd = internalSessionMap.get(internalId);
if (sessionMd != null) {
if (!StringUtils.isEmpty(perSessionUpdatesConfiguration)) {
TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(perSessionUpdatesConfiguration));
if (!rateLimits.tryConsume()) {
if (blacklistedSessions.putIfAbsent(externalId, sessionRef) == null) {
log.info("[{}][{}][{}] Failed to process session update. Max session updates limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), externalId);
synchronized (sessionMd) {
sessionMd.session.sendMessage(new TextMessage("{\"subscriptionId\":" + subscriptionId + ", \"errorCode\":" + ThingsboardErrorCode.TOO_MANY_UPDATES.getErrorCode() + ", \"errorMsg\":\"Too many updates!\"}"));
}
}
return;
} else {
log.debug("[{}][{}][{}] Session is no longer blacklisted.", sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), externalId);
blacklistedSessions.remove(externalId);
}
}
synchronized (sessionMd) {
sessionMd.session.sendMessage(new TextMessage(msg));
}
@ -163,14 +211,14 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
@Override
public void close(TelemetryWebSocketSessionRef sessionRef) throws IOException {
public void close(TelemetryWebSocketSessionRef sessionRef, CloseStatus reason) throws IOException {
String externalId = sessionRef.getSessionId();
log.debug("[{}] Processing close request", externalId);
String internalId = externalSessionMap.get(externalId);
if (internalId != null) {
SessionMetaData sessionMd = internalSessionMap.get(internalId);
if (sessionMd != null) {
sessionMd.session.close(CloseStatus.NORMAL);
sessionMd.session.close(reason);
} else {
log.warn("[{}][{}] Failed to find session by internal id", externalId, internalId);
}
@ -179,4 +227,96 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
}
private boolean checkLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) throws Exception {
String sessionId = session.getId();
if (maxSessionsPerTenant > 0) {
Set<String> tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet());
synchronized (tenantSessions) {
if (tenantSessions.size() < maxSessionsPerTenant) {
tenantSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max tenant sessions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), sessionId);
session.close(CloseStatus.POLICY_VIOLATION.withReason("Max tenant sessions limit reached!"));
return false;
}
}
}
if (sessionRef.getSecurityCtx().isCustomerUser()) {
if (maxSessionsPerCustomer > 0) {
Set<String> customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet());
synchronized (customerSessions) {
if (customerSessions.size() < maxSessionsPerCustomer) {
customerSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max customer sessions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), sessionId);
session.close(CloseStatus.POLICY_VIOLATION.withReason("Max customer sessions limit reached"));
return false;
}
}
}
if (maxSessionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (regularUserSessions) {
if (regularUserSessions.size() < maxSessionsPerRegularUser) {
regularUserSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max regular user sessions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), sessionId);
session.close(CloseStatus.POLICY_VIOLATION.withReason("Max regular user sessions limit reached"));
return false;
}
}
}
if (maxSessionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (publicUserSessions) {
if (publicUserSessions.size() < maxSessionsPerPublicUser) {
publicUserSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max public user sessions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), sessionId);
session.close(CloseStatus.POLICY_VIOLATION.withReason("Max public user sessions limit reached"));
return false;
}
}
}
}
return true;
}
private void cleanupLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) {
String sessionId = session.getId();
perSessionUpdateLimits.remove(sessionRef.getSessionId());
blacklistedSessions.remove(sessionRef.getSessionId());
if (maxSessionsPerTenant > 0) {
Set<String> tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet());
synchronized (tenantSessions) {
tenantSessions.remove(sessionId);
}
}
if (sessionRef.getSecurityCtx().isCustomerUser()) {
if (maxSessionsPerCustomer > 0) {
Set<String> customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet());
synchronized (customerSessions) {
customerSessions.remove(sessionId);
}
}
if (maxSessionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (regularUserSessions) {
regularUserSessions.remove(sessionId);
}
}
if (maxSessionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (publicUserSessions) {
publicUserSessions.remove(sessionId);
}
}
}
}
}

15
application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java

@ -25,8 +25,10 @@ import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.web.access.AccessDeniedHandler;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.service.security.exception.AuthMethodNotSupportedException;
import org.thingsboard.server.service.security.exception.JwtExpiredTokenException;
@ -34,6 +36,7 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
@Component
@Slf4j
public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
@ -62,6 +65,8 @@ public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
if (exception instanceof ThingsboardException) {
handleThingsboardException((ThingsboardException) exception, response);
} else if (exception instanceof TbRateLimitsException) {
handleRateLimitException(response, (TbRateLimitsException) exception);
} else if (exception instanceof AccessDeniedException) {
handleAccessDeniedException(response);
} else if (exception instanceof AuthenticationException) {
@ -77,6 +82,7 @@ public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
}
}
private void handleThingsboardException(ThingsboardException thingsboardException, HttpServletResponse response) throws IOException {
ThingsboardErrorCode errorCode = thingsboardException.getErrorCode();
@ -110,6 +116,15 @@ public class ThingsboardErrorResponseHandler implements AccessDeniedHandler {
mapper.writeValue(response.getWriter(), ThingsboardErrorResponse.of(thingsboardException.getMessage(), errorCode, status));
}
private void handleRateLimitException(HttpServletResponse response, TbRateLimitsException exception) throws IOException {
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
String message = "Too many requests for current " + exception.getEntityType().name().toLowerCase() + "!";
mapper.writeValue(response.getWriter(),
ThingsboardErrorResponse.of(message,
ThingsboardErrorCode.TOO_MANY_REQUESTS, HttpStatus.TOO_MANY_REQUESTS));
}
private void handleAccessDeniedException(HttpServletResponse response) throws IOException {
response.setStatus(HttpStatus.FORBIDDEN.value());
mapper.writeValue(response.getWriter(),

147
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java

@ -23,12 +23,17 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.WebSocketSession;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
@ -42,6 +47,7 @@ import org.thingsboard.server.service.security.AccessValidator;
import org.thingsboard.server.service.security.ValidationCallback;
import org.thingsboard.server.service.security.ValidationResult;
import org.thingsboard.server.service.security.ValidationResultCode;
import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.telemetry.cmd.AttributesSubscriptionCmd;
import org.thingsboard.server.service.telemetry.cmd.GetHistoryCmd;
import org.thingsboard.server.service.telemetry.cmd.SubscriptionCmd;
@ -64,6 +70,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@ -72,6 +79,10 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -82,8 +93,8 @@ import java.util.stream.Collectors;
@Slf4j
public class DefaultTelemetryWebSocketService implements TelemetryWebSocketService {
public static final int DEFAULT_LIMIT = 100;
public static final Aggregation DEFAULT_AGGREGATION = Aggregation.NONE;
private static final int DEFAULT_LIMIT = 100;
private static final Aggregation DEFAULT_AGGREGATION = Aggregation.NONE;
private static final int UNKNOWN_SUBSCRIPTION_ID = 0;
private static final String PROCESSING_MSG = "[{}] Processing: {}";
private static final ObjectMapper jsonMapper = new ObjectMapper();
@ -108,11 +119,25 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
@Autowired
private TimeseriesService tsService;
@Value("${server.ws.limits.max_subscriptions_per_tenant:0}")
private int maxSubscriptionsPerTenant;
@Value("${server.ws.limits.max_subscriptions_per_customer:0}")
private int maxSubscriptionsPerCustomer;
@Value("${server.ws.limits.max_subscriptions_per_regular_user:0}")
private int maxSubscriptionsPerRegularUser;
@Value("${server.ws.limits.max_subscriptions_per_public_user:0}")
private int maxSubscriptionsPerPublicUser;
private ConcurrentMap<TenantId, Set<String>> tenantSubscriptionsMap = new ConcurrentHashMap<>();
private ConcurrentMap<CustomerId, Set<String>> customerSubscriptionsMap = new ConcurrentHashMap<>();
private ConcurrentMap<UserId, Set<String>> regularUserSubscriptionsMap = new ConcurrentHashMap<>();
private ConcurrentMap<UserId, Set<String>> publicUserSubscriptionsMap = new ConcurrentHashMap<>();
private ExecutorService executor;
@PostConstruct
public void initExecutor() {
executor = Executors.newSingleThreadExecutor();
executor = new ThreadPoolExecutor(0, 50, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
}
@PreDestroy
@ -136,6 +161,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
case CLOSED:
wsSessionsMap.remove(sessionId);
subscriptionManager.cleanupLocalWsSessionSubscriptions(sessionRef, sessionId);
processSessionClose(sessionRef);
break;
}
}
@ -150,10 +176,18 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
TelemetryPluginCmdsWrapper cmdsWrapper = jsonMapper.readValue(msg, TelemetryPluginCmdsWrapper.class);
if (cmdsWrapper != null) {
if (cmdsWrapper.getAttrSubCmds() != null) {
cmdsWrapper.getAttrSubCmds().forEach(cmd -> handleWsAttributesSubscriptionCmd(sessionRef, cmd));
cmdsWrapper.getAttrSubCmds().forEach(cmd -> {
if (processSubscription(sessionRef, cmd)) {
handleWsAttributesSubscriptionCmd(sessionRef, cmd);
}
});
}
if (cmdsWrapper.getTsSubCmds() != null) {
cmdsWrapper.getTsSubCmds().forEach(cmd -> handleWsTimeseriesSubscriptionCmd(sessionRef, cmd));
cmdsWrapper.getTsSubCmds().forEach(cmd -> {
if (processSubscription(sessionRef, cmd)) {
handleWsTimeseriesSubscriptionCmd(sessionRef, cmd);
}
});
}
if (cmdsWrapper.getHistoryCmds() != null) {
cmdsWrapper.getHistoryCmds().forEach(cmd -> handleWsHistoryCmd(sessionRef, cmd));
@ -174,6 +208,105 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
}
}
private void processSessionClose(TelemetryWebSocketSessionRef sessionRef) {
String sessionId = "[" + sessionRef.getSessionId() + "]";
if (maxSubscriptionsPerTenant > 0) {
Set<String> tenantSubscriptions = tenantSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet());
synchronized (tenantSubscriptions) {
tenantSubscriptions.removeIf(subId -> subId.startsWith(sessionId));
}
}
if (sessionRef.getSecurityCtx().isCustomerUser()) {
if (maxSubscriptionsPerCustomer > 0) {
Set<String> customerSessions = customerSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet());
synchronized (customerSessions) {
customerSessions.removeIf(subId -> subId.startsWith(sessionId));
}
}
if (maxSubscriptionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> regularUserSessions = regularUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (regularUserSessions) {
regularUserSessions.removeIf(subId -> subId.startsWith(sessionId));
}
}
if (maxSubscriptionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> publicUserSessions = publicUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (publicUserSessions) {
publicUserSessions.removeIf(subId -> subId.startsWith(sessionId));
}
}
}
}
private boolean processSubscription(TelemetryWebSocketSessionRef sessionRef, SubscriptionCmd cmd) {
String subId = "[" + sessionRef.getSessionId() + "]:[" + cmd.getCmdId() + "]";
try {
if (maxSubscriptionsPerTenant > 0) {
Set<String> tenantSubscriptions = tenantSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet());
synchronized (tenantSubscriptions) {
if (cmd.isUnsubscribe()) {
tenantSubscriptions.remove(subId);
} else if (tenantSubscriptions.size() < maxSubscriptionsPerTenant) {
tenantSubscriptions.add(subId);
} else {
log.info("[{}][{}][{}] Failed to start subscription. Max tenant subscriptions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), subId);
msgEndpoint.close(sessionRef, CloseStatus.POLICY_VIOLATION.withReason("Max tenant subscriptions limit reached!"));
return false;
}
}
}
if (sessionRef.getSecurityCtx().isCustomerUser()) {
if (maxSubscriptionsPerCustomer > 0) {
Set<String> customerSessions = customerSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet());
synchronized (customerSessions) {
if (cmd.isUnsubscribe()) {
customerSessions.remove(subId);
} else if (customerSessions.size() < maxSubscriptionsPerCustomer) {
customerSessions.add(subId);
} else {
log.info("[{}][{}][{}] Failed to start subscription. Max customer subscriptions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), subId);
msgEndpoint.close(sessionRef, CloseStatus.POLICY_VIOLATION.withReason("Max customer subscriptions limit reached"));
return false;
}
}
}
if (maxSubscriptionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> regularUserSessions = regularUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (regularUserSessions) {
if (regularUserSessions.size() < maxSubscriptionsPerRegularUser) {
regularUserSessions.add(subId);
} else {
log.info("[{}][{}][{}] Failed to start subscription. Max regular user subscriptions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), subId);
msgEndpoint.close(sessionRef, CloseStatus.POLICY_VIOLATION.withReason("Max regular user subscriptions limit reached"));
return false;
}
}
}
if (maxSubscriptionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> publicUserSessions = publicUserSubscriptionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (publicUserSessions) {
if (publicUserSessions.size() < maxSubscriptionsPerPublicUser) {
publicUserSessions.add(subId);
} else {
log.info("[{}][{}][{}] Failed to start subscription. Max public user subscriptions limit reached"
, sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), subId);
msgEndpoint.close(sessionRef, CloseStatus.POLICY_VIOLATION.withReason("Max public user subscriptions limit reached"));
return false;
}
}
}
}
} catch (IOException e) {
log.warn("[{}] Failed to send session close: {}", sessionRef.getSessionId(), e);
return false;
}
return true;
}
private void handleWsAttributesSubscriptionCmd(TelemetryWebSocketSessionRef sessionRef, AttributesSubscriptionCmd cmd) {
String sessionId = sessionRef.getSessionId();
log.debug("[{}] Processing: {}", sessionId, cmd);
@ -216,7 +349,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
public void onFailure(Throwable e) {
log.error(FAILED_TO_FETCH_ATTRIBUTES, e);
SubscriptionUpdate update;
if (UnauthorizedException.class.isInstance(e)) {
if (e instanceof UnauthorizedException) {
update = new SubscriptionUpdate(cmd.getCmdId(), SubscriptionErrorCode.UNAUTHORIZED,
SubscriptionErrorCode.UNAUTHORIZED.getDefaultMsg());
} else {
@ -449,7 +582,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
private void sendWsMsg(TelemetryWebSocketSessionRef sessionRef, SubscriptionUpdate update) {
try {
msgEndpoint.send(sessionRef, jsonMapper.writeValueAsString(update));
msgEndpoint.send(sessionRef, update.getSubscriptionId(), jsonMapper.writeValueAsString(update));
} catch (JsonProcessingException e) {
log.warn("[{}] Failed to encode reply: {}", sessionRef.getSessionId(), update, e);
} catch (IOException e) {

7
application/src/main/java/org/thingsboard/server/service/telemetry/TelemetryWebSocketMsgEndpoint.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.service.telemetry;
import org.springframework.web.socket.CloseStatus;
import java.io.IOException;
/**
@ -22,8 +24,7 @@ import java.io.IOException;
*/
public interface TelemetryWebSocketMsgEndpoint {
void send(TelemetryWebSocketSessionRef sessionRef, String msg) throws IOException;
void close(TelemetryWebSocketSessionRef sessionRef) throws IOException;
void send(TelemetryWebSocketSessionRef sessionRef, int subscriptionId, String msg) throws IOException;
void close(TelemetryWebSocketSessionRef sessionRef, CloseStatus withReason) throws IOException;
}

1
application/src/main/java/org/thingsboard/server/service/transport/RemoteRuleEngineTransportService.java

@ -112,7 +112,6 @@ public class RemoteRuleEngineTransportService implements RuleEngineTransportServ
public void init() {
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<ToTransportMsg> notificationsProducerBuilder = TBKafkaProducerTemplate.builder();
notificationsProducerBuilder.settings(kafkaSettings);
notificationsProducerBuilder.defaultTopic(notificationsTopic);
notificationsProducerBuilder.encoder(new ToTransportMsgEncoder());
notificationsProducer = notificationsProducerBuilder.build();

6
application/src/main/java/org/thingsboard/server/service/transport/RemoteTransportApiService.java

@ -29,6 +29,7 @@ import org.thingsboard.server.kafka.*;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ -43,8 +44,6 @@ public class RemoteTransportApiService {
@Value("${transport.remote.transport_api.requests_topic}")
private String transportApiRequestsTopic;
@Value("${transport.remote.transport_api.responses_topic}")
private String transportApiResponsesTopic;
@Value("${transport.remote.transport_api.max_pending_requests}")
private int maxPendingRequests;
@Value("${transport.remote.transport_api.request_timeout}")
@ -69,11 +68,10 @@ public class RemoteTransportApiService {
@PostConstruct
public void init() {
this.transportCallbackExecutor = new ThreadPoolExecutor(0, 100, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
this.transportCallbackExecutor = new ThreadPoolExecutor(0, 100, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
TBKafkaProducerTemplate.TBKafkaProducerTemplateBuilder<TransportApiResponseMsg> responseBuilder = TBKafkaProducerTemplate.builder();
responseBuilder.settings(kafkaSettings);
responseBuilder.defaultTopic(transportApiResponsesTopic);
responseBuilder.encoder(new TransportApiResponseEncoder());
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<TransportApiRequestMsg> requestBuilder = TBKafkaConsumerTemplate.builder();

23
application/src/main/resources/thingsboard.yml

@ -32,6 +32,26 @@ server:
# Alias that identifies the key in the key store
key-alias: "${SSL_KEY_ALIAS:tomcat}"
log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:true}"
ws:
limits:
# Limit the amount of sessions and subscriptions available on each server. Put values to zero to disable particular limitation
max_sessions_per_tenant: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_TENANT:0}"
max_sessions_per_customer: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_CUSTOMER:0}"
max_sessions_per_regular_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_REGULAR_USER:0}"
max_sessions_per_public_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_PUBLIC_USER:0}"
max_subscriptions_per_tenant: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_TENANT:0}"
max_subscriptions_per_customer: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_CUSTOMER:0}"
max_subscriptions_per_regular_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_REGULAR_USER:0}"
max_subscriptions_per_public_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_PUBLIC_USER:0}"
max_updates_per_session: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_UPDATES_PER_SESSION:300:1,3000:60}"
rest:
limits:
tenant:
enabled: "${TB_SERVER_REST_LIMITS_TENANT_ENABLED:false}"
configuration: "${TB_SERVER_REST_LIMITS_TENANT_CONFIGURATION:100:1,2000:60}"
customer:
enabled: "${TB_SERVER_REST_LIMITS_CUSTOMER_ENABLED:false}"
configuration: "${TB_SERVER_REST_LIMITS_CUSTOMER_CONFIGURATION:50:1,1000:60}"
# Zookeeper connection parameters. Used for service discovery.
zk:
@ -55,6 +75,8 @@ rpc:
# Clustering properties related to consistent-hashing. See architecture docs for more details.
cluster:
# Unique id for this node (autogenerated if empty)
node_id: "${CLUSTER_NODE_ID:}"
# Name of hash function used for consistent hash ring.
hash_function_name: "${CLUSTER_HASH_FUNCTION_NAME:murmur3_128}"
# Amount of virtual nodes in consistent hash ring.
@ -392,7 +414,6 @@ transport:
remote:
transport_api:
requests_topic: "${TB_TRANSPORT_API_REQUEST_TOPIC:tb.transport.api.requests}"
responses_topic: "${TB_TRANSPORT_API_RESPONSE_TOPIC:tb.transport.api.responses}"
max_pending_requests: "${TB_TRANSPORT_MAX_PENDING_REQUESTS:10000}"
request_timeout: "${TB_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}"
request_poll_interval: "${TB_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}"

4
common/data/src/main/java/org/thingsboard/server/common/data/exception/ThingsboardErrorCode.java

@ -25,7 +25,9 @@ public enum ThingsboardErrorCode {
PERMISSION_DENIED(20),
INVALID_ARGUMENTS(30),
BAD_REQUEST_PARAMS(31),
ITEM_NOT_FOUND(32);
ITEM_NOT_FOUND(32),
TOO_MANY_REQUESTS(33),
TOO_MANY_UPDATES(34);
private int errorCode;

8
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TbRateLimitsException.java → common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimitsException.java

@ -13,17 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.service;
package org.thingsboard.server.common.msg.tools;
import lombok.Getter;
import org.thingsboard.server.common.data.EntityType;
/**
* Created by ashvayka on 22.10.18.
*/
public class TbRateLimitsException extends Exception {
public class TbRateLimitsException extends RuntimeException {
@Getter
private final EntityType entityType;
TbRateLimitsException(EntityType entityType) {
public TbRateLimitsException(EntityType entityType) {
this.entityType = entityType;
}
}

33
common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaAdmin.java

@ -15,15 +15,17 @@
*/
package org.thingsboard.server.kafka;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.CreateTopicsResult;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.admin.*;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.KafkaFuture;
import java.time.Duration;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* Created by ashvayka on 24.09.18.
@ -36,7 +38,32 @@ public class TBKafkaAdmin {
client = AdminClient.create(settings.toProps());
}
public void waitForTopic(String topic, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException {
synchronized (this) {
long timeoutExpiredMs = System.currentTimeMillis() + timeoutUnit.toMillis(timeout);
while (!topicExists(topic)) {
long waitMs = timeoutExpiredMs - System.currentTimeMillis();
if (waitMs <= 0) {
throw new TimeoutException("Timeout occurred while waiting for topic [" + topic + "] to be available!");
} else {
wait(1000);
}
}
}
}
public CreateTopicsResult createTopic(NewTopic topic){
return client.createTopics(Collections.singletonList(topic));
}
private boolean topicExists(String topic) throws InterruptedException {
KafkaFuture<TopicDescription> topicDescriptionFuture = client.describeTopics(Collections.singleton(topic)).values().get(topic);
try {
topicDescriptionFuture.get();
return true;
} catch (ExecutionException e) {
return false;
}
}
}

32
common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java

@ -20,14 +20,17 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.admin.CreateTopicsResult;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.admin.TopicDescription;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.KafkaFuture;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.errors.TopicExistsException;
import org.apache.kafka.common.header.Header;
import org.springframework.util.StringUtils;
import java.util.List;
import java.util.Properties;
@ -35,6 +38,7 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
/**
* Created by ashvayka on 24.09.18.
@ -71,21 +75,19 @@ public class TBKafkaProducerTemplate<T> {
}
public void init() {
try {
TBKafkaAdmin admin = new TBKafkaAdmin(this.settings);
CreateTopicsResult result = admin.createTopic(new NewTopic(defaultTopic, 100, (short) 1));
result.all().get();
} catch (Exception e) {
if ((e instanceof TopicExistsException) || (e.getCause() != null && e.getCause() instanceof TopicExistsException)) {
log.trace("[{}] Topic already exists.", defaultTopic);
} else {
log.info("[{}] Failed to create topic: {}", defaultTopic, e.getMessage(), e);
this.partitionInfoMap = new ConcurrentHashMap<>();
if (!StringUtils.isEmpty(defaultTopic)) {
try {
TBKafkaAdmin admin = new TBKafkaAdmin(this.settings);
admin.waitForTopic(defaultTopic, 30, TimeUnit.SECONDS);
log.info("[{}] Topic exists.", defaultTopic);
} catch (Exception e) {
log.info("[{}] Failed to wait for topic: {}", defaultTopic, e.getMessage(), e);
throw new RuntimeException(e);
}
//Maybe this should not be cached, but we don't plan to change size of partitions
this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic));
}
//Maybe this should not be cached, but we don't plan to change size of partitions
this.partitionInfoMap = new ConcurrentHashMap<>();
this.partitionInfoMap.putIfAbsent(defaultTopic, producer.partitionsFor(defaultTopic));
}
T enrich(T value, String responseTopic, UUID requestId) {
@ -105,7 +107,11 @@ public class TBKafkaProducerTemplate<T> {
}
public Future<RecordMetadata> send(String key, T value, Long timestamp, Iterable<Header> headers, Callback callback) {
return send(this.defaultTopic, key, value, timestamp, headers, callback);
if (!StringUtils.isEmpty(this.defaultTopic)) {
return send(this.defaultTopic, key, value, timestamp, headers, callback);
} else {
throw new RuntimeException("Failed to send message! Default topic is not specified!");
}
}
public Future<RecordMetadata> send(String topic, String key, T value, Iterable<Header> headers, Callback callback) {

4
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/AbstractTransportService.java

@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.common.transport.SessionMsgListener;
import org.thingsboard.server.common.transport.TransportService;
import org.thingsboard.server.common.transport.TransportServiceCallback;
@ -31,6 +32,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
@ -276,7 +278,7 @@ public abstract class AbstractTransportService implements TransportService {
new TbRateLimits(perDevicesLimitsConf);
}
this.schedulerExecutor = Executors.newSingleThreadScheduledExecutor();
this.transportCallbackExecutor = new ThreadPoolExecutor(0, 20, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
this.transportCallbackExecutor = new ThreadPoolExecutor(0, 20, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
this.schedulerExecutor.scheduleAtFixedRate(this::checkInactivityAndReportActivity, sessionReportTimeout, sessionReportTimeout, TimeUnit.MILLISECONDS);
}

12
dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java

@ -214,7 +214,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
if (customer == null) {
throw new DataValidationException("Can't unassign dashboards from non-existent customer!");
}
new CustomerDashboardsUnassigner(customer).removeEntities(customer);
new CustomerDashboardsUnassigner(customer).removeEntities(tenantId, customer);
}
@Override
@ -225,7 +225,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
if (customer == null) {
throw new DataValidationException("Can't update dashboards for non-existent customer!");
}
new CustomerDashboardsUpdater(customer).removeEntities(customer);
new CustomerDashboardsUpdater(customer).removeEntities(tenantId, customer);
}
private DataValidator<Dashboard> dashboardValidator =
@ -269,7 +269,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
}
@Override
protected List<DashboardInfo> findEntities(Customer customer, TimePageLink pageLink) {
protected List<DashboardInfo> findEntities(TenantId tenantId, Customer customer, TimePageLink pageLink) {
try {
return dashboardInfoDao.findDashboardsByTenantIdAndCustomerId(customer.getTenantId().getId(), customer.getId().getId(), pageLink).get();
} catch (InterruptedException | ExecutionException e) {
@ -279,7 +279,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
}
@Override
protected void removeEntity(DashboardInfo entity) {
protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
unassignDashboardFromCustomer(customer.getTenantId(), new DashboardId(entity.getUuidId()), this.customer.getId());
}
@ -294,7 +294,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
}
@Override
protected List<DashboardInfo> findEntities(Customer customer, TimePageLink pageLink) {
protected List<DashboardInfo> findEntities(TenantId tenantId, Customer customer, TimePageLink pageLink) {
try {
return dashboardInfoDao.findDashboardsByTenantIdAndCustomerId(customer.getTenantId().getId(), customer.getId().getId(), pageLink).get();
} catch (InterruptedException | ExecutionException e) {
@ -304,7 +304,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
}
@Override
protected void removeEntity(DashboardInfo entity) {
protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
updateAssignedCustomer(customer.getTenantId(), new DashboardId(entity.getUuidId()), this.customer);
}

8
dao/src/main/java/org/thingsboard/server/dao/model/nosql/EntityViewEntity.java

@ -60,10 +60,6 @@ public class EntityViewEntity implements SearchTextEntity<EntityView> {
@Column(name = ID_PROPERTY)
private UUID id;
@Enumerated(EnumType.STRING)
@Column(name = ENTITY_TYPE_PROPERTY)
private EntityType entityType;
@PartitionKey(value = 1)
@Column(name = ModelConstants.ENTITY_VIEW_TENANT_ID_PROPERTY)
private UUID tenantId;
@ -76,6 +72,10 @@ public class EntityViewEntity implements SearchTextEntity<EntityView> {
@Column(name = DEVICE_TYPE_PROPERTY)
private String type;
@Enumerated(EnumType.STRING)
@Column(name = ENTITY_TYPE_PROPERTY)
private EntityType entityType;
@Column(name = ModelConstants.ENTITY_VIEW_ENTITY_ID_PROPERTY)
private UUID entityId;

3
dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraBufferedRateExecutor.java

@ -17,7 +17,6 @@ package org.thingsboard.server.dao.nosql;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.ResultSetFuture;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@ -26,7 +25,6 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.util.AbstractBufferedRateExecutor;
import org.thingsboard.server.dao.util.AsyncTaskContext;
import org.thingsboard.server.dao.util.NoSqlAnyDao;
@ -34,7 +32,6 @@ import org.thingsboard.server.dao.util.NoSqlAnyDao;
import javax.annotation.PreDestroy;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* Created by ashvayka on 24.10.18.

11
dao/src/main/java/org/thingsboard/server/dao/service/TimePaginatedRemover.java

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.service;
import org.thingsboard.server.common.data.id.IdBased;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.TimePageLink;
import java.util.List;
@ -25,13 +26,13 @@ public abstract class TimePaginatedRemover<I, D extends IdBased<?>> {
private static final int DEFAULT_LIMIT = 100;
public void removeEntities(I id) {
public void removeEntities(TenantId tenantId, I id) {
TimePageLink pageLink = new TimePageLink(DEFAULT_LIMIT);
boolean hasNext = true;
while (hasNext) {
List<D> entities = findEntities(id, pageLink);
List<D> entities = findEntities(tenantId, id, pageLink);
for (D entity : entities) {
removeEntity(entity);
removeEntity(tenantId, entity);
}
hasNext = entities.size() == pageLink.getLimit();
if (hasNext) {
@ -42,8 +43,8 @@ public abstract class TimePaginatedRemover<I, D extends IdBased<?>> {
}
}
protected abstract List<D> findEntities(I id, TimePageLink pageLink);
protected abstract List<D> findEntities(TenantId tenantId, I id, TimePageLink pageLink);
protected abstract void removeEntity(D entity);
protected abstract void removeEntity(TenantId tenantId, D entity);
}

7
dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java

@ -20,12 +20,10 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import javax.annotation.Nullable;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
@ -33,11 +31,12 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/**
* Created by ashvayka on 24.10.18.
@ -73,7 +72,7 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
this.concurrencyLimit = concurrencyLimit;
this.queue = new LinkedBlockingDeque<>(queueLimit);
this.dispatcherExecutor = Executors.newFixedThreadPool(dispatcherThreads);
this.callbackExecutor = Executors.newFixedThreadPool(callbackThreads);
this.callbackExecutor = new ThreadPoolExecutor(callbackThreads, 50, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor();
this.perTenantLimitsEnabled = perTenantLimitsEnabled;
this.perTenantLimitsConfiguration = perTenantLimitsConfiguration;

2
dao/src/main/resources/sql/schema-entities.sql

@ -236,7 +236,7 @@ CREATE TABLE IF NOT EXISTS entity_view (
customer_id varchar(31),
type varchar(255),
name varchar(255),
keys varchar(255),
keys varchar(10000000),
start_ts bigint,
end_ts bigint,
search_text varchar(255),

7
docker/docker-compose.yml

@ -56,6 +56,7 @@ services:
max-file: "30"
environment:
TB_HOST: tb1
CLUSTER_NODE_ID: tb1
env_file:
- tb-node.env
volumes:
@ -77,6 +78,7 @@ services:
max-file: "30"
environment:
TB_HOST: tb2
CLUSTER_NODE_ID: tb2
env_file:
- tb-node.env
volumes:
@ -93,6 +95,7 @@ services:
- "1883"
environment:
TB_HOST: tb-mqtt-transport1
CLUSTER_NODE_ID: tb-mqtt-transport1
env_file:
- tb-mqtt-transport.env
volumes:
@ -107,6 +110,7 @@ services:
- "1883"
environment:
TB_HOST: tb-mqtt-transport2
CLUSTER_NODE_ID: tb-mqtt-transport2
env_file:
- tb-mqtt-transport.env
volumes:
@ -121,6 +125,7 @@ services:
- "8081"
environment:
TB_HOST: tb-http-transport1
CLUSTER_NODE_ID: tb-http-transport1
env_file:
- tb-http-transport.env
volumes:
@ -135,6 +140,7 @@ services:
- "8081"
environment:
TB_HOST: tb-http-transport2
CLUSTER_NODE_ID: tb-http-transport2
env_file:
- tb-http-transport.env
volumes:
@ -149,6 +155,7 @@ services:
- "5683:5683/udp"
environment:
TB_HOST: tb-coap-transport
CLUSTER_NODE_ID: tb-coap-transport
env_file:
- tb-coap-transport.env
volumes:

2
docker/kafka.env

@ -4,7 +4,7 @@ KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092
KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE
KAFKA_CREATE_TOPICS=js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1
KAFKA_CREATE_TOPICS=js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600
KAFKA_AUTO_CREATE_TOPICS_ENABLE=false
KAFKA_LOG_RETENTION_BYTES=1073741824
KAFKA_LOG_SEGMENT_BYTES=268435456

2
docker/tb-node.env

@ -8,3 +8,5 @@ JS_EVALUATOR=remote
TRANSPORT_TYPE=remote
CACHE_TYPE=redis
REDIS_HOST=redis
HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false

43
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java

@ -66,7 +66,7 @@ public class MqttClientTest extends AbstractContainerTest {
WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS);
MqttClient mqttClient = getMqttClient(deviceCredentials, null);
mqttClient.publish("v1/devices/me/telemetry", Unpooled.wrappedBuffer(createPayload().toString().getBytes()));
mqttClient.publish("v1/devices/me/telemetry", Unpooled.wrappedBuffer(createPayload().toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
@ -93,7 +93,7 @@ public class MqttClientTest extends AbstractContainerTest {
WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS);
MqttClient mqttClient = getMqttClient(deviceCredentials, null);
mqttClient.publish("v1/devices/me/telemetry", Unpooled.wrappedBuffer(createPayload(ts).toString().getBytes()));
mqttClient.publish("v1/devices/me/telemetry", Unpooled.wrappedBuffer(createPayload(ts).toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
@ -123,7 +123,7 @@ public class MqttClientTest extends AbstractContainerTest {
clientAttributes.addProperty("attr2", true);
clientAttributes.addProperty("attr3", 42.0);
clientAttributes.addProperty("attr4", 73);
mqttClient.publish("v1/devices/me/attributes", Unpooled.wrappedBuffer(clientAttributes.toString().getBytes()));
mqttClient.publish("v1/devices/me/attributes", Unpooled.wrappedBuffer(clientAttributes.toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
@ -146,6 +146,7 @@ public class MqttClientTest extends AbstractContainerTest {
Device device = createDevice("mqtt_");
DeviceCredentials deviceCredentials = restClient.getCredentials(device.getId());
WsClient wsClient = subscribeToWebSocket(device.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS);
MqttMessageListener listener = new MqttMessageListener();
MqttClient mqttClient = getMqttClient(deviceCredentials, listener);
@ -153,7 +154,17 @@ public class MqttClientTest extends AbstractContainerTest {
JsonObject clientAttributes = new JsonObject();
String clientAttributeValue = RandomStringUtils.randomAlphanumeric(8);
clientAttributes.addProperty("clientAttr", clientAttributeValue);
mqttClient.publish("v1/devices/me/attributes", Unpooled.wrappedBuffer(clientAttributes.toString().getBytes()));
mqttClient.publish("v1/devices/me/attributes", Unpooled.wrappedBuffer(clientAttributes.toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received ws telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
Assert.assertEquals(1, actualLatestTelemetry.getData().size());
Assert.assertEquals(Sets.newHashSet("clientAttr"),
actualLatestTelemetry.getLatestValues().keySet());
Assert.assertTrue(verify(actualLatestTelemetry, "clientAttr", clientAttributeValue));
// Add a new shared attribute
JsonObject sharedAttributes = new JsonObject();
@ -166,12 +177,16 @@ public class MqttClientTest extends AbstractContainerTest {
Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful());
// Subscribe to attributes response
mqttClient.on("v1/devices/me/attributes/response/+", listener, MqttQoS.AT_LEAST_ONCE);
mqttClient.on("v1/devices/me/attributes/response/+", listener, MqttQoS.AT_LEAST_ONCE).get();
// Wait until subscription is processed
TimeUnit.SECONDS.sleep(3);
// Request attributes
JsonObject request = new JsonObject();
request.addProperty("clientKeys", "clientAttr");
request.addProperty("sharedKeys", "sharedAttr");
mqttClient.publish("v1/devices/me/attributes/request/" + new Random().nextInt(100), Unpooled.wrappedBuffer(request.toString().getBytes()));
mqttClient.publish("v1/devices/me/attributes/request/" + new Random().nextInt(100), Unpooled.wrappedBuffer(request.toString().getBytes())).get();
MqttEvent event = listener.getEvents().poll(10, TimeUnit.SECONDS);
AttributesResponse attributes = mapper.readValue(Objects.requireNonNull(event).getMessage(), AttributesResponse.class);
log.info("Received telemetry: {}", attributes);
@ -193,7 +208,10 @@ public class MqttClientTest extends AbstractContainerTest {
MqttMessageListener listener = new MqttMessageListener();
MqttClient mqttClient = getMqttClient(deviceCredentials, listener);
mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE);
mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE).get();
// Wait until subscription is processed
TimeUnit.SECONDS.sleep(3);
String sharedAttributeName = "sharedAttr";
@ -236,7 +254,10 @@ public class MqttClientTest extends AbstractContainerTest {
MqttMessageListener listener = new MqttMessageListener();
MqttClient mqttClient = getMqttClient(deviceCredentials, listener);
mqttClient.on("v1/devices/me/rpc/request/+", listener, MqttQoS.AT_LEAST_ONCE);
mqttClient.on("v1/devices/me/rpc/request/+", listener, MqttQoS.AT_LEAST_ONCE).get();
// Wait until subscription is processed
TimeUnit.SECONDS.sleep(3);
// Send an RPC from the server
JsonObject serverRpcPayload = new JsonObject();
@ -263,7 +284,7 @@ public class MqttClientTest extends AbstractContainerTest {
JsonObject clientResponse = new JsonObject();
clientResponse.addProperty("response", "someResponse");
// Send a response to the server's RPC request
mqttClient.publish("v1/devices/me/rpc/response/" + requestId, Unpooled.wrappedBuffer(clientResponse.toString().getBytes()));
mqttClient.publish("v1/devices/me/rpc/response/" + requestId, Unpooled.wrappedBuffer(clientResponse.toString().getBytes())).get();
ResponseEntity serverResponse = future.get(5, TimeUnit.SECONDS);
Assert.assertTrue(serverResponse.getStatusCode().is2xxSuccessful());
@ -280,7 +301,7 @@ public class MqttClientTest extends AbstractContainerTest {
MqttMessageListener listener = new MqttMessageListener();
MqttClient mqttClient = getMqttClient(deviceCredentials, listener);
mqttClient.on("v1/devices/me/rpc/request/+", listener, MqttQoS.AT_LEAST_ONCE);
mqttClient.on("v1/devices/me/rpc/request/+", listener, MqttQoS.AT_LEAST_ONCE).get();
// Get the default rule chain id to make it root again after test finished
RuleChainId defaultRuleChainId = getDefaultRuleChainId();
@ -294,7 +315,7 @@ public class MqttClientTest extends AbstractContainerTest {
clientRequest.addProperty("method", "getResponse");
clientRequest.addProperty("params", true);
Integer requestId = 42;
mqttClient.publish("v1/devices/me/rpc/request/" + requestId, Unpooled.wrappedBuffer(clientRequest.toString().getBytes()));
mqttClient.publish("v1/devices/me/rpc/request/" + requestId, Unpooled.wrappedBuffer(clientRequest.toString().getBytes())).get();
// Check the response from the server
TimeUnit.SECONDS.sleep(1);

4
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java

@ -390,7 +390,9 @@ final class MqttClientImpl implements MqttClient {
}
public void onSuccessfulReconnect() {
callback.onSuccessfulReconnect();
if (callback != null) {
callback.onSuccessfulReconnect();
}
}

2
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java

@ -34,6 +34,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
/**
* Created by ashvayka on 13.01.18.
@ -98,4 +99,5 @@ public interface TbContext {
ScriptEngine createJsScriptEngine(String script, String... argNames);
String getNodeId();
}

101
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNode.java

@ -0,0 +1,101 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.action;
import com.datastax.driver.core.utils.UUIDs;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.rule.engine.api.*;
import org.thingsboard.rule.engine.api.util.TbNodeUtils;
import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.session.SessionMsgType;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS;
@Slf4j
@RuleNode(
type = ComponentType.ACTION,
name = "message count",
configClazz = TbMsgCountNodeConfiguration.class,
nodeDescription = "Count incoming messages",
nodeDetails = "Count incoming messages for specified interval and produces POST_TELEMETRY_REQUEST msg with messages count",
icon = "functions",
uiResources = {"static/rulenode/rulenode-core-config.js"},
configDirective = "tbActionNodeMsgCountConfig"
)
public class TbMsgCountNode implements TbNode {
private static final String TB_MSG_COUNT_NODE_MSG = "TbMsgCountNodeMsg";
private AtomicLong messagesProcessed = new AtomicLong(0);
private final Gson gson = new Gson();
private UUID nextTickId;
private long delay;
private String telemetryPrefix;
private long lastScheduledTs;
@Override
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
TbMsgCountNodeConfiguration config = TbNodeUtils.convert(configuration, TbMsgCountNodeConfiguration.class);
this.delay = TimeUnit.SECONDS.toMillis(config.getInterval());
this.telemetryPrefix = config.getTelemetryPrefix();
scheduleTickMsg(ctx);
}
@Override
public void onMsg(TbContext ctx, TbMsg msg) {
if (msg.getType().equals(TB_MSG_COUNT_NODE_MSG) && msg.getId().equals(nextTickId)) {
JsonObject telemetryJson = new JsonObject();
telemetryJson.addProperty(this.telemetryPrefix + "_" + ctx.getNodeId(), messagesProcessed.longValue());
messagesProcessed = new AtomicLong(0);
TbMsgMetaData metaData = new TbMsgMetaData();
metaData.putValue("delta", Long.toString(System.currentTimeMillis() - lastScheduledTs + delay));
TbMsg tbMsg = new TbMsg(UUIDs.timeBased(), SessionMsgType.POST_TELEMETRY_REQUEST.name(), ctx.getTenantId(), metaData, TbMsgDataType.JSON, gson.toJson(telemetryJson), null, null, 0L);
ctx.tellNext(tbMsg, SUCCESS);
scheduleTickMsg(ctx);
} else {
messagesProcessed.incrementAndGet();
}
}
private void scheduleTickMsg(TbContext ctx) {
long curTs = System.currentTimeMillis();
if (lastScheduledTs == 0L) {
lastScheduledTs = curTs;
}
lastScheduledTs = lastScheduledTs + delay;
long curDelay = Math.max(0L, (lastScheduledTs - curTs));
TbMsg tickMsg = ctx.newMsg(TB_MSG_COUNT_NODE_MSG, ctx.getSelfId(), new TbMsgMetaData(), "");
nextTickId = tickMsg.getId();
ctx.tellSelf(tickMsg, curDelay);
}
@Override
public void destroy() {
}
}

34
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbMsgCountNodeConfiguration.java

@ -0,0 +1,34 @@
/**
* Copyright © 2016-2018 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.action;
import lombok.Data;
import org.thingsboard.rule.engine.api.NodeConfiguration;
@Data
public class TbMsgCountNodeConfiguration implements NodeConfiguration<TbMsgCountNodeConfiguration> {
private String telemetryPrefix;
private int interval;
@Override
public TbMsgCountNodeConfiguration defaultConfiguration() {
TbMsgCountNodeConfiguration configuration = new TbMsgCountNodeConfiguration();
configuration.setInterval(1);
configuration.setTelemetryPrefix("messageCount");
return configuration;
}
}

6
rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js

File diff suppressed because one or more lines are too long

7
transport/coap/src/main/resources/tb-coap-transport.yml

@ -17,7 +17,12 @@
spring.main.web-environment: false
spring.main.web-application-type: none
# MQTT server parameters
# Clustering properties
cluster:
# Unique id for this node (autogenerated if empty)
node_id: "${CLUSTER_NODE_ID:}"
# COAP server parameters
transport:
coap:
bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}"

5
transport/http/src/main/resources/tb-http-transport.yml

@ -20,6 +20,11 @@ server:
# Server bind port
port: "${HTTP_BIND_PORT:8081}"
# Clustering properties
cluster:
# Unique id for this node (autogenerated if empty)
node_id: "${CLUSTER_NODE_ID:}"
# HTTP server parameters
transport:
http:

5
transport/mqtt/src/main/resources/tb-mqtt-transport.yml

@ -17,6 +17,11 @@
spring.main.web-environment: false
spring.main.web-application-type: none
# Clustering properties
cluster:
# Unique id for this node (autogenerated if empty)
node_id: "${CLUSTER_NODE_ID:}"
# MQTT server parameters
transport:
mqtt:

29
ui/src/app/api/telemetry-websocket.service.js

@ -26,7 +26,7 @@ const WS_IDLE_TIMEOUT = 90000;
const MAX_PUBLISH_COMMANDS = 10;
/*@ngInject*/
function TelemetryWebsocketService($rootScope, $websocket, $timeout, $window, types, userService) {
function TelemetryWebsocketService($rootScope, $websocket, $timeout, $window, $mdUtil, $log, toast, types, userService) {
var isOpening = false,
isOpened = false,
@ -111,7 +111,11 @@ function TelemetryWebsocketService($rootScope, $websocket, $timeout, $window, ty
}
}
function onError (/*message*/) {
function onError (errorEvent) {
if (errorEvent) {
//showWsError(0, errorEvent);
$log.warn('WebSocket error event', errorEvent);
}
isOpening = false;
}
@ -137,7 +141,10 @@ function TelemetryWebsocketService($rootScope, $websocket, $timeout, $window, ty
}
}
function onClose () {
function onClose (closeEvent) {
if (closeEvent && closeEvent.code > 1000 && closeEvent.code !== 1006) {
showWsError(closeEvent.code, closeEvent.reason);
}
isOpening = false;
isOpened = false;
if (isActive) {
@ -162,7 +169,9 @@ function TelemetryWebsocketService($rootScope, $websocket, $timeout, $window, ty
function onMessage (message) {
if (message.data) {
var data = angular.fromJson(message.data);
if (data.subscriptionId) {
if (data.errorCode) {
showWsError(data.errorCode, data.errorMsg);
} else if (data.subscriptionId) {
var subscriber = subscribers[data.subscriptionId];
if (subscriber && data) {
var keys = fetchKeys(data.subscriptionId);
@ -182,6 +191,18 @@ function TelemetryWebsocketService($rootScope, $websocket, $timeout, $window, ty
checkToClose();
}
function showWsError(errorCode, errorMsg) {
var message = 'WebSocket Error: ';
if (errorMsg) {
message += errorMsg;
} else {
message += "error code - " + errorCode + ".";
}
$mdUtil.nextTick(function () {
toast.showError(message);
});
}
function fetchKeys(subscriptionId) {
var command = commands[subscriptionId];
if (command && command.keys && command.keys.length > 0) {

4
ui/src/app/common/types.constant.js

@ -23,7 +23,9 @@ export default angular.module('thingsboard.types', [])
permissionDenied: 20,
invalidArguments: 30,
badRequestParams: 31,
itemNotFound: 32
itemNotFound: 32,
tooManyRequests: 33,
tooManyUpdates: 34
},
entryPoints: {
login: "/api/auth/login",

3
ui/src/app/services/toast.scss

@ -16,18 +16,21 @@
md-toast.tb-info-toast .md-toast-content {
height: 100%;
max-height: 100%;
padding: 18px;
font-size: 18px;
}
md-toast.tb-success-toast .md-toast-content {
height: 100%;
max-height: 100%;
font-size: 18px !important;
background-color: #008000;
}
md-toast.tb-error-toast .md-toast-content {
height: 100%;
max-height: 100%;
font-size: 18px !important;
background-color: #800000;
}

Loading…
Cancel
Save