Browse Source

Merge with develop/2.5.6

release-2.5
Igor Kulikov 6 years ago
parent
commit
19584eaaa2
  1. 3
      application/src/main/resources/thingsboard.yml
  2. 4
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  3. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java
  4. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  5. 34
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
  6. 16
      dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java
  7. 43
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  8. 30
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraPartitionCacheKey.java
  9. 42
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraTsPartitionsCache.java
  10. 9
      dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java
  11. 125
      dao/src/test/java/org/thingsboard/server/dao/nosql/CassandraPartitionsCacheTest.java
  12. 2
      dao/src/test/resources/cassandra-test.properties
  13. 14
      msa/js-executor/package-lock.json
  14. 2
      msa/js-executor/package.json
  15. 2
      msa/web-ui/package-lock.json
  16. 2
      msa/web-ui/package.json
  17. 43
      ui/package-lock.json
  18. 2
      ui/package.json

3
application/src/main/resources/thingsboard.yml

@ -223,8 +223,9 @@ cassandra:
read_consistency_level: "${CASSANDRA_READ_CONSISTENCY_LEVEL:ONE}" read_consistency_level: "${CASSANDRA_READ_CONSISTENCY_LEVEL:ONE}"
write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}" write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}"
default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}" default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}"
# Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS,INDEFINITE # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS, INDEFINITE
ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}"
ts_key_value_ttl: "${TS_KV_TTL:0}" ts_key_value_ttl: "${TS_KV_TTL:0}"
events_ttl: "${TS_EVENTS_TTL:0}" events_ttl: "${TS_EVENTS_TTL:0}"
# Specify TTL of debug log in seconds. The current value corresponds to one week # Specify TTL of debug log in seconds. The current value corresponds to one week

4
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -152,7 +152,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.settings(kafkaSettings); consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(ruleEngineSettings.getTopic()); consumerBuilder.topic(ruleEngineSettings.getTopic());
consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("re-" + queueName + "-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.admin(ruleEngineAdmin);
return consumerBuilder.build(); return consumerBuilder.build();
@ -176,7 +176,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
consumerBuilder.settings(kafkaSettings); consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.topic(coreSettings.getTopic());
consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.clientId("monolith-core-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("monolith-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("monolith-core-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin); consumerBuilder.admin(coreAdmin);
return consumerBuilder.build(); return consumerBuilder.build();

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@ -146,7 +146,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
consumerBuilder.settings(kafkaSettings); consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(coreSettings.getTopic()); consumerBuilder.topic(coreSettings.getTopic());
consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.clientId("tb-core-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("tb-core-node-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("tb-core-node");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(coreAdmin); consumerBuilder.admin(coreAdmin);
return consumerBuilder.build(); return consumerBuilder.build();

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java

@ -141,7 +141,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
consumerBuilder.settings(kafkaSettings); consumerBuilder.settings(kafkaSettings);
consumerBuilder.topic(ruleEngineSettings.getTopic()); consumerBuilder.topic(ruleEngineSettings.getTopic());
consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId());
consumerBuilder.groupId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId()); consumerBuilder.groupId("re-" + queueName + "-consumer");
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()));
consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.admin(ruleEngineAdmin);
return consumerBuilder.build(); return consumerBuilder.build();

34
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java

@ -357,37 +357,37 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
log.info("[{}] Processing connect msg for client: {}!", sessionId, msg.payload().clientIdentifier()); log.info("[{}] Processing connect msg for client: {}!", sessionId, msg.payload().clientIdentifier());
X509Certificate cert; X509Certificate cert;
if (sslHandler != null && (cert = getX509Certificate()) != null) { if (sslHandler != null && (cert = getX509Certificate()) != null) {
processX509CertConnect(ctx, cert); processX509CertConnect(ctx, cert, msg);
} else { } else {
processAuthTokenConnect(ctx, msg); processAuthTokenConnect(ctx, msg);
} }
} }
private void processAuthTokenConnect(ChannelHandlerContext ctx, MqttConnectMessage msg) { private void processAuthTokenConnect(ChannelHandlerContext ctx, MqttConnectMessage connectMessage) {
String userName = msg.payload().userName(); String userName = connectMessage.payload().userName();
log.info("[{}] Processing connect msg for client with user name: {}!", sessionId, userName); log.info("[{}] Processing connect msg for client with user name: {}!", sessionId, userName);
if (StringUtils.isEmpty(userName)) { if (StringUtils.isEmpty(userName)) {
ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_BAD_USER_NAME_OR_PASSWORD)); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_BAD_USER_NAME_OR_PASSWORD, connectMessage));
ctx.close(); ctx.close();
} else { } else {
transportService.process(ValidateDeviceTokenRequestMsg.newBuilder().setToken(userName).build(), transportService.process(ValidateDeviceTokenRequestMsg.newBuilder().setToken(userName).build(),
new TransportServiceCallback<ValidateDeviceCredentialsResponseMsg>() { new TransportServiceCallback<ValidateDeviceCredentialsResponseMsg>() {
@Override @Override
public void onSuccess(ValidateDeviceCredentialsResponseMsg msg) { public void onSuccess(ValidateDeviceCredentialsResponseMsg msg) {
onValidateDeviceResponse(msg, ctx); onValidateDeviceResponse(msg, ctx, connectMessage);
} }
@Override @Override
public void onError(Throwable e) { public void onError(Throwable e) {
log.trace("[{}] Failed to process credentials: {}", address, userName, e); log.trace("[{}] Failed to process credentials: {}", address, userName, e);
ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE)); ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE, connectMessage));
ctx.close(); ctx.close();
} }
}); });
} }
} }
private void processX509CertConnect(ChannelHandlerContext ctx, X509Certificate cert) { private void processX509CertConnect(ChannelHandlerContext ctx, X509Certificate cert, MqttConnectMessage connectMessage) {
try { try {
if(!context.isSkipValidityCheckForClientCert()){ if(!context.isSkipValidityCheckForClientCert()){
cert.checkValidity(); cert.checkValidity();
@ -398,18 +398,18 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
new TransportServiceCallback<ValidateDeviceCredentialsResponseMsg>() { new TransportServiceCallback<ValidateDeviceCredentialsResponseMsg>() {
@Override @Override
public void onSuccess(ValidateDeviceCredentialsResponseMsg msg) { public void onSuccess(ValidateDeviceCredentialsResponseMsg msg) {
onValidateDeviceResponse(msg, ctx); onValidateDeviceResponse(msg, ctx, connectMessage);
} }
@Override @Override
public void onError(Throwable e) { public void onError(Throwable e) {
log.trace("[{}] Failed to process credentials: {}", address, sha3Hash, e); log.trace("[{}] Failed to process credentials: {}", address, sha3Hash, e);
ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE)); ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE, connectMessage));
ctx.close(); ctx.close();
} }
}); });
} catch (Exception e) { } catch (Exception e) {
ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED)); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED, connectMessage));
ctx.close(); ctx.close();
} }
} }
@ -433,11 +433,11 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
doDisconnect(); doDisconnect();
} }
private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode) { private MqttConnAckMessage createMqttConnAckMsg(MqttConnectReturnCode returnCode, MqttConnectMessage msg) {
MqttFixedHeader mqttFixedHeader = MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(CONNACK, false, AT_MOST_ONCE, false, 0); new MqttFixedHeader(CONNACK, false, AT_MOST_ONCE, false, 0);
MqttConnAckVariableHeader mqttConnAckVariableHeader = MqttConnAckVariableHeader mqttConnAckVariableHeader =
new MqttConnAckVariableHeader(returnCode, true); new MqttConnAckVariableHeader(returnCode, !msg.variableHeader().isCleanSession());
return new MqttConnAckMessage(mqttFixedHeader, mqttConnAckVariableHeader); return new MqttConnAckMessage(mqttFixedHeader, mqttConnAckVariableHeader);
} }
@ -513,9 +513,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
} }
} }
private void onValidateDeviceResponse(ValidateDeviceCredentialsResponseMsg msg, ChannelHandlerContext ctx) { private void onValidateDeviceResponse(ValidateDeviceCredentialsResponseMsg msg, ChannelHandlerContext ctx, MqttConnectMessage connectMessage) {
if (!msg.hasDeviceInfo()) { if (!msg.hasDeviceInfo()) {
ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED)); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_REFUSED_NOT_AUTHORIZED, connectMessage));
ctx.close(); ctx.close();
} else { } else {
deviceSessionCtx.setDeviceInfo(msg.getDeviceInfo()); deviceSessionCtx.setDeviceInfo(msg.getDeviceInfo());
@ -532,17 +532,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
.build(); .build();
transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), new TransportServiceCallback<Void>() { transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), new TransportServiceCallback<Void>() {
@Override @Override
public void onSuccess(Void msg) { public void onSuccess(Void response) {
transportService.registerAsyncSession(sessionInfo, MqttTransportHandler.this); transportService.registerAsyncSession(sessionInfo, MqttTransportHandler.this);
checkGatewaySession(); checkGatewaySession();
ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED)); ctx.writeAndFlush(createMqttConnAckMsg(CONNECTION_ACCEPTED, connectMessage));
log.info("[{}] Client connected!", sessionId); log.info("[{}] Client connected!", sessionId);
} }
@Override @Override
public void onError(Throwable e) { public void onError(Throwable e) {
log.warn("[{}] Failed to submit session event", sessionId, e); log.warn("[{}] Failed to submit session event", sessionId, e);
ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE)); ctx.writeAndFlush(createMqttConnAckMsg(MqttConnectReturnCode.CONNECTION_REFUSED_SERVER_UNAVAILABLE, connectMessage));
ctx.close(); ctx.close();
} }
}); });

16
dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java

@ -17,7 +17,6 @@ package org.thingsboard.server.dao.audit;
import com.datastax.driver.core.utils.UUIDs; import com.datastax.driver.core.utils.UUIDs;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
@ -48,6 +47,7 @@ import org.thingsboard.server.dao.audit.sink.AuditLogSink;
import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.util.mapping.JacksonUtil;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.io.StringWriter; import java.io.StringWriter;
@ -61,8 +61,6 @@ import static org.thingsboard.server.dao.service.Validator.validateId;
@ConditionalOnProperty(prefix = "audit-log", value = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "audit-log", value = "enabled", havingValue = "true")
public class AuditLogServiceImpl implements AuditLogService { public class AuditLogServiceImpl implements AuditLogService {
private static final ObjectMapper objectMapper = new ObjectMapper();
private static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; private static final String INCORRECT_TENANT_ID = "Incorrect tenantId ";
private static final int INSERTS_PER_ENTRY = 3; private static final int INSERTS_PER_ENTRY = 3;
@ -159,7 +157,7 @@ public class AuditLogServiceImpl implements AuditLogService {
private <E extends HasName, I extends EntityId> JsonNode constructActionData(I entityId, E entity, private <E extends HasName, I extends EntityId> JsonNode constructActionData(I entityId, E entity,
ActionType actionType, ActionType actionType,
Object... additionalInfo) { Object... additionalInfo) {
ObjectNode actionData = objectMapper.createObjectNode(); ObjectNode actionData = JacksonUtil.newObjectNode();
switch (actionType) { switch (actionType) {
case ADDED: case ADDED:
case UPDATED: case UPDATED:
@ -168,7 +166,7 @@ public class AuditLogServiceImpl implements AuditLogService {
case RELATIONS_DELETED: case RELATIONS_DELETED:
case ASSIGNED_TO_TENANT: case ASSIGNED_TO_TENANT:
if (entity != null) { if (entity != null) {
ObjectNode entityNode = objectMapper.valueToTree(entity); ObjectNode entityNode = (ObjectNode) JacksonUtil.valueToTree(entity);
if (entityId.getEntityType() == EntityType.DASHBOARD) { if (entityId.getEntityType() == EntityType.DASHBOARD) {
entityNode.put("configuration", ""); entityNode.put("configuration", "");
} }
@ -177,7 +175,7 @@ public class AuditLogServiceImpl implements AuditLogService {
if (entityId.getEntityType() == EntityType.RULE_CHAIN) { if (entityId.getEntityType() == EntityType.RULE_CHAIN) {
RuleChainMetaData ruleChainMetaData = extractParameter(RuleChainMetaData.class, additionalInfo); RuleChainMetaData ruleChainMetaData = extractParameter(RuleChainMetaData.class, additionalInfo);
if (ruleChainMetaData != null) { if (ruleChainMetaData != null) {
ObjectNode ruleChainMetaDataNode = objectMapper.valueToTree(ruleChainMetaData); ObjectNode ruleChainMetaDataNode = (ObjectNode) JacksonUtil.valueToTree(ruleChainMetaData);
actionData.set("metadata", ruleChainMetaDataNode); actionData.set("metadata", ruleChainMetaDataNode);
} }
} }
@ -194,7 +192,7 @@ public class AuditLogServiceImpl implements AuditLogService {
String scope = extractParameter(String.class, 0, additionalInfo); String scope = extractParameter(String.class, 0, additionalInfo);
List<AttributeKvEntry> attributes = extractParameter(List.class, 1, additionalInfo); List<AttributeKvEntry> attributes = extractParameter(List.class, 1, additionalInfo);
actionData.put("scope", scope); actionData.put("scope", scope);
ObjectNode attrsNode = objectMapper.createObjectNode(); ObjectNode attrsNode = JacksonUtil.newObjectNode();
if (attributes != null) { if (attributes != null) {
for (AttributeKvEntry attr : attributes) { for (AttributeKvEntry attr : attributes) {
attrsNode.put(attr.getKey(), attr.getValueAsString()); attrsNode.put(attr.getKey(), attr.getValueAsString());
@ -225,7 +223,7 @@ public class AuditLogServiceImpl implements AuditLogService {
case CREDENTIALS_UPDATED: case CREDENTIALS_UPDATED:
actionData.put("entityId", entityId.toString()); actionData.put("entityId", entityId.toString());
DeviceCredentials deviceCredentials = extractParameter(DeviceCredentials.class, additionalInfo); DeviceCredentials deviceCredentials = extractParameter(DeviceCredentials.class, additionalInfo);
actionData.set("credentials", objectMapper.valueToTree(deviceCredentials)); actionData.set("credentials", JacksonUtil.valueToTree(deviceCredentials));
break; break;
case ASSIGNED_TO_CUSTOMER: case ASSIGNED_TO_CUSTOMER:
strEntityId = extractParameter(String.class, 0, additionalInfo); strEntityId = extractParameter(String.class, 0, additionalInfo);
@ -246,7 +244,7 @@ public class AuditLogServiceImpl implements AuditLogService {
case RELATION_ADD_OR_UPDATE: case RELATION_ADD_OR_UPDATE:
case RELATION_DELETED: case RELATION_DELETED:
EntityRelation relation = extractParameter(EntityRelation.class, 0, additionalInfo); EntityRelation relation = extractParameter(EntityRelation.class, 0, additionalInfo);
actionData.set("relation", objectMapper.valueToTree(relation)); actionData.set("relation", JacksonUtil.valueToTree(relation));
break; break;
case LOGIN: case LOGIN:
case LOGOUT: case LOGOUT:

43
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@ -29,6 +29,7 @@ import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -66,6 +67,7 @@ import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -88,12 +90,17 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
public static final String DESC_ORDER = "DESC"; public static final String DESC_ORDER = "DESC";
private static List<Long> FIXED_PARTITION = Arrays.asList(new Long[]{0L}); private static List<Long> FIXED_PARTITION = Arrays.asList(new Long[]{0L});
private CassandraTsPartitionsCache cassandraTsPartitionsCache;
@Autowired @Autowired
private Environment environment; private Environment environment;
@Value("${cassandra.query.ts_key_value_partitioning}") @Value("${cassandra.query.ts_key_value_partitioning}")
private String partitioning; private String partitioning;
@Value("${cassandra.query.ts_key_value_partitions_max_cache_size:100000}")
private long partitionsCacheSize;
@Value("${cassandra.query.ts_key_value_ttl}") @Value("${cassandra.query.ts_key_value_ttl}")
private long systemTtl; private long systemTtl;
@ -126,6 +133,9 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning); Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning);
if (partition.isPresent()) { if (partition.isPresent()) {
tsFormat = partition.get(); tsFormat = partition.get();
if (!isFixedPartitioning() && partitionsCacheSize > 0) {
cassandraTsPartitionsCache = new CassandraTsPartitionsCache(partitionsCacheSize);
}
} else { } else {
log.warn("Incorrect configuration of partitioning {}", partitioning); log.warn("Incorrect configuration of partitioning {}", partitioning);
throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!"); throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
@ -390,6 +400,39 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
} }
ttl = computeTtl(ttl); ttl = computeTtl(ttl);
long partition = toPartitionTs(tsKvEntryTs); long partition = toPartitionTs(tsKvEntryTs);
if (cassandraTsPartitionsCache == null) {
return doSavePartition(tenantId, entityId, key, ttl, partition);
} else {
CassandraPartitionCacheKey partitionSearchKey = new CassandraPartitionCacheKey(entityId, key, partition);
if (!cassandraTsPartitionsCache.has(partitionSearchKey)) {
ListenableFuture<Void> result = doSavePartition(tenantId, entityId, key, ttl, partition);
Futures.addCallback(result, new CacheCallback<>(partitionSearchKey), MoreExecutors.directExecutor());
return result;
} else {
return Futures.immediateFuture(null);
}
}
}
private class CacheCallback<Void> implements FutureCallback<Void> {
private final CassandraPartitionCacheKey key;
private CacheCallback(CassandraPartitionCacheKey key) {
this.key = key;
}
@Override
public void onSuccess(Void result) {
cassandraTsPartitionsCache.put(key);
}
@Override
public void onFailure(Throwable t) {
}
}
private ListenableFuture<Void> doSavePartition(TenantId tenantId, EntityId entityId, String key, long ttl, long partition) {
log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key); log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key);
BoundStatement stmt = (ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt()).bind(); BoundStatement stmt = (ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt()).bind();
stmt = stmt.setString(0, entityId.getEntityType().name()) stmt = stmt.setString(0, entityId.getEntityType().name())

30
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraPartitionCacheKey.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.timeseries;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.thingsboard.server.common.data.id.EntityId;
@Data
@AllArgsConstructor
public class CassandraPartitionCacheKey {
private EntityId entityId;
private String key;
private long partition;
}

42
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraTsPartitionsCache.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.timeseries;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.concurrent.CompletableFuture;
public class CassandraTsPartitionsCache {
private AsyncLoadingCache<CassandraPartitionCacheKey, Boolean> partitionsCache;
public CassandraTsPartitionsCache(long maxCacheSize) {
this.partitionsCache = Caffeine.newBuilder()
.maximumSize(maxCacheSize)
.buildAsync(key -> {
throw new IllegalStateException("'get' methods calls are not supported!");
});
}
public boolean has(CassandraPartitionCacheKey key) {
return partitionsCache.getIfPresent(key) != null;
}
public void put(CassandraPartitionCacheKey key) {
partitionsCache.put(key, CompletableFuture.completedFuture(true));
}
}

9
dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.util.mapping;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException; import java.io.IOException;
@ -66,7 +67,15 @@ public class JacksonUtil {
} }
} }
public static ObjectNode newObjectNode(){
return OBJECT_MAPPER.createObjectNode();
}
public static <T> T clone(T value) { public static <T> T clone(T value) {
return fromString(toString(value), (Class<T>) value.getClass()); return fromString(toString(value), (Class<T>) value.getClass());
} }
public static <T> JsonNode valueToTree(T value) {
return OBJECT_MAPPER.valueToTree(value);
}
} }

125
dao/src/test/java/org/thingsboard/server/dao/nosql/CassandraPartitionsCacheTest.java

@ -0,0 +1,125 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.nosql;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.CodecRegistry;
import com.datastax.driver.core.Configuration;
import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.google.common.util.concurrent.Futures;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import org.springframework.core.env.Environment;
import org.springframework.test.util.ReflectionTestUtils;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.cassandra.CassandraCluster;
import org.thingsboard.server.dao.timeseries.CassandraBaseTimeseriesDao;
import java.util.UUID;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class CassandraPartitionsCacheTest {
private CassandraBaseTimeseriesDao cassandraBaseTimeseriesDao;
@Mock
private Environment environment;
@Mock
private CassandraBufferedRateExecutor rateLimiter;
@Mock
private CassandraCluster cluster;
@Mock
private Session session;
@Mock
private Cluster sessionCluster;
@Mock
private Configuration configuration;
@Mock
private PreparedStatement preparedStatement;
@Mock
private BoundStatement boundStatement;
@Before
public void setUp() {
when(cluster.getDefaultReadConsistencyLevel()).thenReturn(ConsistencyLevel.ONE);
when(cluster.getDefaultWriteConsistencyLevel()).thenReturn(ConsistencyLevel.ONE);
when(cluster.getSession()).thenReturn(session);
when(session.getCluster()).thenReturn(sessionCluster);
when(sessionCluster.getConfiguration()).thenReturn(configuration);
when(configuration.getCodecRegistry()).thenReturn(CodecRegistry.DEFAULT_INSTANCE);
when(session.prepare(anyString())).thenReturn(preparedStatement);
when(preparedStatement.bind()).thenReturn(boundStatement);
when(boundStatement.setString(anyInt(), anyString())).thenReturn(boundStatement);
when(boundStatement.setUUID(anyInt(), any(UUID.class))).thenReturn(boundStatement);
when(boundStatement.setLong(anyInt(), anyLong())).thenReturn(boundStatement);
when(boundStatement.setInt(anyInt(), anyInt())).thenReturn(boundStatement);
cassandraBaseTimeseriesDao = spy(new CassandraBaseTimeseriesDao());
ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "partitioning", "MONTHS");
ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "partitionsCacheSize", 100000);
ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "systemTtl", 0);
ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "setNullValuesEnabled", false);
ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "environment", environment);
ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "rateLimiter", rateLimiter);
ReflectionTestUtils.setField(cassandraBaseTimeseriesDao, "cluster", cluster);
doReturn(Futures.immediateFuture(null)).when(cassandraBaseTimeseriesDao).getFuture(any(ResultSetFuture.class), any());
}
@Test
public void testPartitionSave() throws Exception {
cassandraBaseTimeseriesDao.init();
UUID id = UUID.randomUUID();
TenantId tenantId = new TenantId(id);
long tsKvEntryTs = System.currentTimeMillis();
for (int i = 0; i < 50000; i++) {
cassandraBaseTimeseriesDao.savePartition(tenantId, tenantId, tsKvEntryTs, "test" + i, 0);
}
for (int i = 0; i < 60000; i++) {
cassandraBaseTimeseriesDao.savePartition(tenantId, tenantId, tsKvEntryTs, "test" + i, 0);
}
verify(cassandraBaseTimeseriesDao, times(60000)).executeAsyncWrite(any(TenantId.class), any(Statement.class));
}
}

2
dao/src/test/resources/cassandra-test.properties

@ -46,6 +46,8 @@ cassandra.query.default_fetch_size=2000
cassandra.query.ts_key_value_partitioning=HOURS cassandra.query.ts_key_value_partitioning=HOURS
cassandra.query.ts_key_value_partitions_max_cache_size=100000
cassandra.query.ts_key_value_ttl=0 cassandra.query.ts_key_value_ttl=0
cassandra.query.debug_events_ttl=604800 cassandra.query.debug_events_ttl=604800

14
msa/js-executor/package-lock.json

@ -1,6 +1,6 @@
{ {
"name": "thingsboard-js-executor", "name": "thingsboard-js-executor",
"version": "2.5.5", "version": "2.5.6",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {
@ -1826,7 +1826,7 @@
}, },
"get-stream": { "get-stream": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
"integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
"dev": true "dev": true
}, },
@ -1936,7 +1936,7 @@
}, },
"got": { "got": {
"version": "6.7.1", "version": "6.7.1",
"resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", "resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz",
"integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=",
"dev": true, "dev": true,
"requires": { "requires": {
@ -2275,7 +2275,7 @@
}, },
"is-obj": { "is-obj": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
"integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=",
"dev": true "dev": true
}, },
@ -2917,7 +2917,7 @@
}, },
"path-is-absolute": { "path-is-absolute": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"dev": true "dev": true
}, },
@ -3474,7 +3474,7 @@
}, },
"safe-regex": { "safe-regex": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
"integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=",
"dev": true, "dev": true,
"requires": { "requires": {
@ -3832,7 +3832,7 @@
}, },
"strip-eof": { "strip-eof": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
"integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
"dev": true "dev": true
}, },

2
msa/js-executor/package.json

@ -1,7 +1,7 @@
{ {
"name": "thingsboard-js-executor", "name": "thingsboard-js-executor",
"private": true, "private": true,
"version": "2.5.5", "version": "2.5.6",
"description": "ThingsBoard JavaScript Executor Microservice", "description": "ThingsBoard JavaScript Executor Microservice",
"main": "server.js", "main": "server.js",
"bin": "server.js", "bin": "server.js",

2
msa/web-ui/package-lock.json

@ -1,6 +1,6 @@
{ {
"name": "thingsboard-web-ui", "name": "thingsboard-web-ui",
"version": "2.5.5", "version": "2.5.6",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {

2
msa/web-ui/package.json

@ -1,7 +1,7 @@
{ {
"name": "thingsboard-web-ui", "name": "thingsboard-web-ui",
"private": true, "private": true,
"version": "2.5.5", "version": "2.5.6",
"description": "ThingsBoard Web UI Microservice", "description": "ThingsBoard Web UI Microservice",
"main": "server.js", "main": "server.js",
"bin": "server.js", "bin": "server.js",

43
ui/package-lock.json

@ -1,6 +1,6 @@
{ {
"name": "thingsboard", "name": "thingsboard",
"version": "2.5.5", "version": "2.5.6",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {
@ -5238,8 +5238,7 @@
"ansi-regex": { "ansi-regex": {
"version": "2.1.1", "version": "2.1.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"aproba": { "aproba": {
"version": "1.2.0", "version": "1.2.0",
@ -5260,14 +5259,12 @@
"balanced-match": { "balanced-match": {
"version": "1.0.0", "version": "1.0.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"brace-expansion": { "brace-expansion": {
"version": "1.1.11", "version": "1.1.11",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"balanced-match": "^1.0.0", "balanced-match": "^1.0.0",
"concat-map": "0.0.1" "concat-map": "0.0.1"
@ -5282,20 +5279,17 @@
"code-point-at": { "code-point-at": {
"version": "1.1.0", "version": "1.1.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"concat-map": { "concat-map": {
"version": "0.0.1", "version": "0.0.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"console-control-strings": { "console-control-strings": {
"version": "1.1.0", "version": "1.1.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"core-util-is": { "core-util-is": {
"version": "1.0.2", "version": "1.0.2",
@ -5412,8 +5406,7 @@
"inherits": { "inherits": {
"version": "2.0.4", "version": "2.0.4",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"ini": { "ini": {
"version": "1.3.5", "version": "1.3.5",
@ -5425,7 +5418,6 @@
"version": "1.0.0", "version": "1.0.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"number-is-nan": "^1.0.0" "number-is-nan": "^1.0.0"
} }
@ -5440,7 +5432,6 @@
"version": "3.0.4", "version": "3.0.4",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
} }
@ -5448,14 +5439,12 @@
"minimist": { "minimist": {
"version": "0.0.8", "version": "0.0.8",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"minipass": { "minipass": {
"version": "2.9.0", "version": "2.9.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"safe-buffer": "^5.1.2", "safe-buffer": "^5.1.2",
"yallist": "^3.0.0" "yallist": "^3.0.0"
@ -5474,7 +5463,6 @@
"version": "0.5.1", "version": "0.5.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"minimist": "0.0.8" "minimist": "0.0.8"
} }
@ -5564,8 +5552,7 @@
"number-is-nan": { "number-is-nan": {
"version": "1.0.1", "version": "1.0.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"object-assign": { "object-assign": {
"version": "4.1.1", "version": "4.1.1",
@ -5577,7 +5564,6 @@
"version": "1.4.0", "version": "1.4.0",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"wrappy": "1" "wrappy": "1"
} }
@ -5663,8 +5649,7 @@
"safe-buffer": { "safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"safer-buffer": { "safer-buffer": {
"version": "2.1.2", "version": "2.1.2",
@ -5700,7 +5685,6 @@
"version": "1.0.2", "version": "1.0.2",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"code-point-at": "^1.0.0", "code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0", "is-fullwidth-code-point": "^1.0.0",
@ -5720,7 +5704,6 @@
"version": "3.0.1", "version": "3.0.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true,
"optional": true,
"requires": { "requires": {
"ansi-regex": "^2.0.0" "ansi-regex": "^2.0.0"
} }
@ -5764,14 +5747,12 @@
"wrappy": { "wrappy": {
"version": "1.0.2", "version": "1.0.2",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
}, },
"yallist": { "yallist": {
"version": "3.1.1", "version": "3.1.1",
"bundled": true, "bundled": true,
"dev": true, "dev": true
"optional": true
} }
} }
}, },

2
ui/package.json

@ -1,7 +1,7 @@
{ {
"name": "thingsboard", "name": "thingsboard",
"private": true, "private": true,
"version": "2.5.5", "version": "2.5.6",
"description": "ThingsBoard UI", "description": "ThingsBoard UI",
"licenses": [ "licenses": [
{ {

Loading…
Cancel
Save