tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge branch 'feature/entity-data-query' of github.com:thingsboard/thingsboard into feature/entity-data-query

pull/3053/head
Igor Kulikov 6 years ago
parent
bf22c19824 f97d74d6be
commit
0bfbe57479
18 changed files with 352 additions and 288 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 380
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java
  2. 135
      application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java
  3. 9
      application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubscriptionService.java
  4. 21
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java
  5. 3
      application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityHistoryCmd.java
  6. 38
      application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/GetTsCmd.java
  7. 12
      application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/TimeSeriesCmd.java
  8. 2
      application/src/main/resources/thingsboard.yml
  9. 10
      application/src/test/java/org/thingsboard/server/controller/BaseWebsocketApiTest.java
  10. 10
      common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java
  11. 2
      common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java
  12. 6
      common/data/src/main/java/org/thingsboard/server/common/data/query/EntityDataPageLink.java
  13. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  14. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
  15. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java
  16. 2
      dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java
  17. 2
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  18. 2
      dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java

380
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java
View File

@ -19,17 +19,16 @@ import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
@ -40,19 +39,12 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.TsValue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.telemetry.DefaultTelemetryWebSocketService;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef;
@ -60,15 +52,18 @@ import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUnsubscribeCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityHistoryCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.GetTsCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.LatestValueCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.TimeSeriesCmd;
import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode;
import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@ -79,6 +74,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@Slf4j
@ -87,28 +88,14 @@ import java.util.stream.Collectors;
public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubscriptionService {
private static final int DEFAULT_LIMIT = 100;
private final Set<TopicPartitionInfo> currentPartitions = ConcurrentHashMap.newKeySet();
private final Map<String, Map<Integer, TbEntityDataSubCtx>> subscriptionsBySessionId = new ConcurrentHashMap<>();
@Autowired
private TelemetryWebSocketService wsService;
@Autowired
private EntityViewService entityViewService;
@Autowired
private EntityService entityService;
@Autowired
private PartitionService partitionService;
@Autowired
private TbClusterService clusterService;
@Autowired
@Lazy
private SubscriptionManagerService subscriptionManagerService;
@Autowired
@Lazy
private TbLocalSubscriptionService localSubscriptionService;
@ -119,18 +106,38 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
@Autowired
private TbServiceInfoProvider serviceInfoProvider;
@Autowired
@Getter
private DbCallbackExecutorService dbCallbackExecutor;
private ScheduledExecutorService scheduler;
@Value("${database.ts.type}")
private String databaseTsType;
@Value("${server.ws.dynamic_page_link_refresh_interval:6}")
private long dynamicPageLinkRefreshInterval;
@Value("${server.ws.dynamic_page_link_refresh_pool_size:1}")
private int dynamicPageLinkRefreshPoolSize;
private ExecutorService wsCallBackExecutor;
private boolean tsInSqlDB;
private String serviceId;
private AtomicInteger regularQueryInvocationCnt = new AtomicInteger();
private AtomicInteger dynamicQueryInvocationCnt = new AtomicInteger();
private AtomicLong regularQueryTimeSpent = new AtomicLong();
private AtomicLong dynamicQueryTimeSpent = new AtomicLong();
@PostConstruct
public void initExecutor() {
serviceId = serviceInfoProvider.getServiceId();
wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-entity-sub-callback"));
tsInSqlDB = databaseTsType.equalsIgnoreCase("sql") || databaseTsType.equalsIgnoreCase("timescale");
ThreadFactory tbThreadFactory = ThingsBoardThreadFactory.forName("ws-entity-sub-scheduler");
if (dynamicPageLinkRefreshPoolSize == 1) {
scheduler = Executors.newSingleThreadScheduledExecutor(tbThreadFactory);
} else {
scheduler = Executors.newScheduledThreadPool(dynamicPageLinkRefreshPoolSize, tbThreadFactory);
}
}
@PreDestroy
@ -140,45 +147,19 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
}
}
@Override
@EventListener(PartitionChangeEvent.class)
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) {
currentPartitions.clear();
currentPartitions.addAll(partitionChangeEvent.getPartitions());
}
}
@Override
@EventListener(ClusterTopologyChangeEvent.class)
public void onApplicationEvent(ClusterTopologyChangeEvent event) {
if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) {
/*
* If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again.
* Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart.
* Although this is resource consuming operation, it is cheaper than sending ping/pong commands periodically
* It is also cheaper then caching the subscriptions by entity id and then lookup of those caches every time we have new telemetry in SubscriptionManagerService.
* Even if we cache locally the list of active subscriptions by entity id, it is still time consuming operation to get them from cache
* Since number of subscriptions is usually much less then number of devices that are pushing data.
// */
// subscriptionsBySessionId.values().forEach(map -> map.values()
// .forEach(sub -> pushSubscriptionToManagerService(sub, false)));
}
}
@Override
public void handleCmd(TelemetryWebSocketSessionRef session, EntityDataCmd cmd) {
TbEntityDataSubCtx ctx = getSubCtx(session.getSessionId(), cmd.getCmdId());
if (ctx != null) {
log.debug("[{}][{}] Updating existing subscriptions using: {}", session.getSessionId(), cmd.getCmdId(), cmd);
if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null || cmd.getHistoryCmd() != null) {
Collection<Integer> oldSubIds = ctx.clearSubscriptions();
oldSubIds.forEach(subId -> localSubscriptionService.cancelSubscription(serviceId, subId));
clearSubs(ctx);
}
} else {
log.debug("[{}][{}] Creating new subscription using: {}", session.getSessionId(), cmd.getCmdId(), cmd);
ctx = createSubCtx(session, cmd);
}
ctx.setCurrentCmd(cmd);
if (cmd.getQuery() != null) {
if (ctx.getQuery() == null) {
log.debug("[{}][{}] Initializing data using query: {}", session.getSessionId(), cmd.getCmdId(), cmd.getQuery());
@ -197,13 +178,26 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
}
});
}
long start = System.currentTimeMillis();
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, customerId, ctx.getQuery());
long end = System.currentTimeMillis();
regularQueryInvocationCnt.incrementAndGet();
regularQueryTimeSpent.addAndGet(end - start);
if (log.isTraceEnabled()) {
data.getData().forEach(ed -> {
log.trace("[{}][{}] EntityData: {}", session.getSessionId(), cmd.getCmdId(), ed);
});
}
ctx.setData(data);
ctx.cancelRefreshTask();
if (ctx.getQuery().getPageLink().isDynamic()) {
TbEntityDataSubCtx finalCtx = ctx;
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay(
() -> refreshDynamicQuery(tenantId, customerId, finalCtx),
dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS);
finalCtx.setRefreshTask(task);
}
}
ListenableFuture<TbEntityDataSubCtx> historyFuture;
if (cmd.getHistoryCmd() != null) {
@ -233,6 +227,35 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
}, wsCallBackExecutor);
}
private void refreshDynamicQuery(TenantId tenantId, CustomerId customerId, TbEntityDataSubCtx finalCtx) {
try {
long start = System.currentTimeMillis();
Collection<Integer> oldSubIds = finalCtx.update(entityService.findEntityDataByQuery(tenantId, customerId, finalCtx.getQuery()));
long end = System.currentTimeMillis();
dynamicQueryInvocationCnt.incrementAndGet();
dynamicQueryTimeSpent.addAndGet(end - start);
oldSubIds.forEach(subId -> localSubscriptionService.cancelSubscription(serviceId, subId));
} catch (Exception e) {
log.warn("[{}][{}] Failed to refresh query", finalCtx.getSessionId(), finalCtx.getCmdId(), e);
}
}
@Scheduled(fixedDelayString = "${server.ws.dynamic_page_link_stats:10000}")
public void printStats() {
int regularQueryInvocationCntValue = regularQueryInvocationCnt.getAndSet(0);
long regularQueryInvocationTimeValue = regularQueryTimeSpent.getAndSet(0);
int dynamicQueryInvocationCntValue = dynamicQueryInvocationCnt.getAndSet(0);
long dynamicQueryInvocationTimeValue = dynamicQueryTimeSpent.getAndSet(0);
long dynamicQueryCnt = subscriptionsBySessionId.values().stream().map(Map::values).count();
log.info("Stats: regularQueryInvocationCnt = [{}], regularQueryInvocationTime = [{}], dynamicQueryCnt = [{}] dynamicQueryInvocationCnt = [{}], dynamicQueryInvocationTime = [{}]",
regularQueryInvocationCntValue, regularQueryInvocationTimeValue, dynamicQueryCnt, dynamicQueryInvocationCntValue, dynamicQueryInvocationTimeValue);
}
private void clearSubs(TbEntityDataSubCtx ctx) {
Collection<Integer> oldSubIds = ctx.clearSubscriptions();
oldSubIds.forEach(subId -> localSubscriptionService.cancelSubscription(serviceId, subId));
}
private TbEntityDataSubCtx createSubCtx(TelemetryWebSocketSessionRef sessionRef, EntityDataCmd cmd) {
Map<Integer, TbEntityDataSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new HashMap<>());
TbEntityDataSubCtx ctx = new TbEntityDataSubCtx(serviceId, wsService, sessionRef, cmd.getCmdId());
@ -250,49 +273,86 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
}
}
private void handleTimeSeriesCmd(TbEntityDataSubCtx ctx, TimeSeriesCmd cmd) {
List<String> keys = cmd.getKeys();
private ListenableFuture<TbEntityDataSubCtx> handleTimeSeriesCmd(TbEntityDataSubCtx ctx, TimeSeriesCmd cmd) {
log.debug("[{}][{}] Fetching time-series data for last {} ms for keys: ({})", ctx.getSessionId(), ctx.getCmdId(), cmd.getTimeWindow(), cmd.getKeys());
long startTs = cmd.getStartTs();
long endTs = cmd.getStartTs() + cmd.getTimeWindow();
Map<EntityData, ListenableFuture<Map<String, List<TsValue>>>> tsFutures = new HashMap<>();
for (EntityData entityData : ctx.getData().getData()) {
List<ReadTsKvQuery> queries = keys.stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, cmd.getInterval(),
getLimit(cmd.getLimit()), DefaultTelemetryWebSocketService.getAggregation(cmd.getAgg()))).collect(Collectors.toList());
ListenableFuture<List<TsKvEntry>> tsDataFutures = tsService.findAll(ctx.getTenantId(), entityData.getEntityId(), queries);
tsFutures.put(entityData, Futures.transform(tsDataFutures, this::toTsValues, MoreExecutors.directExecutor()));
return handleGetTsCmd(ctx, cmd, true);
}
private ListenableFuture<TbEntityDataSubCtx> handleHistoryCmd(TbEntityDataSubCtx ctx, EntityHistoryCmd cmd) {
log.debug("[{}][{}] Fetching history data for start {} and end {} ms for keys: ({})", ctx.getSessionId(), ctx.getCmdId(), cmd.getStartTs(), cmd.getEndTs(), cmd.getKeys());
return handleGetTsCmd(ctx, cmd, false);
}
private ListenableFuture<TbEntityDataSubCtx> handleGetTsCmd(TbEntityDataSubCtx ctx, GetTsCmd cmd, boolean subscribe) {
List<String> keys = cmd.getKeys();
List<ReadTsKvQuery> finalTsKvQueryList;
List<ReadTsKvQuery> tsKvQueryList = cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery(
key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg()
)).collect(Collectors.toList());
if (cmd.isFetchLatestPreviousPoint()) {
finalTsKvQueryList = new ArrayList<>(tsKvQueryList);
tsKvQueryList.addAll(cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery(
key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg()
)).collect(Collectors.toList()));
} else {
finalTsKvQueryList = tsKvQueryList;
}
Futures.addCallback(Futures.allAsList(tsFutures.values()), new FutureCallback<List<Map<String, List<TsValue>>>>() {
@Override
public void onSuccess(@Nullable List<Map<String, List<TsValue>>> result) {
tsFutures.forEach((key, value) -> {
try {
value.get().forEach((k, v) -> key.getTimeseries().put(k, v.toArray(new TsValue[v.size()])));
} catch (InterruptedException | ExecutionException e) {
log.warn("[{}][{}] Failed to lookup time-series data: {}:{}", ctx.getSessionId(), ctx.getCmdId(), key.getEntityId(), keys, e);
Map<EntityData, ListenableFuture<List<TsKvEntry>>> fetchResultMap = new HashMap<>();
ctx.getData().getData().forEach(entityData -> fetchResultMap.put(entityData,
tsService.findAll(ctx.getTenantId(), entityData.getEntityId(), finalTsKvQueryList)));
return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> {
fetchResultMap.forEach((entityData, future) -> {
Map<String, List<TsValue>> keyData = new LinkedHashMap<>();
cmd.getKeys().forEach(key -> keyData.put(key, new ArrayList<>()));
try {
List<TsKvEntry> entityTsData = future.get();
if (entityTsData != null) {
entityTsData.forEach(entry -> keyData.get(entry.getKey()).add(new TsValue(entry.getTs(), entry.getValueAsString())));
}
});
EntityDataUpdate update;
if (!ctx.isInitialDataSent()) {
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null);
ctx.setInitialDataSent(true);
} else {
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData());
keyData.forEach((k, v) -> entityData.getTimeseries().put(k, v.toArray(new TsValue[v.size()])));
if (cmd.isFetchLatestPreviousPoint()) {
entityData.getTimeseries().values().forEach(dataArray -> {
Arrays.sort(dataArray, (o1, o2) -> Long.compare(o2.getTs(), o1.getTs()));
});
}
} catch (InterruptedException | ExecutionException e) {
log.warn("[{}][{}][{}] Failed to fetch historical data", ctx.getSessionId(), ctx.getCmdId(), entityData.getEntityId(), e);
wsService.sendWsMsg(ctx.getSessionId(),
new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to fetch historical data!"));
}
wsService.sendWsMsg(ctx.getSessionId(), update);
createSubscriptions(ctx, keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).collect(Collectors.toList()), false);
});
EntityDataUpdate update;
if (!ctx.isInitialDataSent()) {
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null);
ctx.setInitialDataSent(true);
} else {
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData());
}
@Override
public void onFailure(Throwable t) {
log.warn("[{}][{}] Failed to process websocket command: {}:{}", ctx.getSessionId(), ctx.getCmdId(), ctx.getQuery(), cmd, t);
wsService.sendWsMsg(ctx.getSessionId(),
new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to process websocket command!"));
wsService.sendWsMsg(ctx.getSessionId(), update);
if (subscribe) {
createSubscriptions(ctx, keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).collect(Collectors.toList()), false);
}
ctx.getData().getData().forEach(ed -> ed.getTimeseries().clear());
return ctx;
}, wsCallBackExecutor);
}
private List<ReadTsKvQuery> getReadTsKvQueries(GetTsCmd cmd) {
List<ReadTsKvQuery> finalTsKvQueryList;
List<ReadTsKvQuery> queries = cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery(key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(),
getLimit(cmd.getLimit()), cmd.getAgg())).collect(Collectors.toList());
if (cmd.isFetchLatestPreviousPoint()) {
finalTsKvQueryList = new ArrayList<>(queries);
finalTsKvQueryList.addAll(cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery(
key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg()
)).collect(Collectors.toList()));
} else {
finalTsKvQueryList = queries;
}
return finalTsKvQueryList;
}
private void handleLatestCmd(TbEntityDataSubCtx ctx, LatestValueCmd latestCmd) {
log.trace("[{}][{}] Going to process latest command: {}", ctx.getSessionId(), ctx.getCmdId(), latestCmd);
//Fetch the latest values for telemetry keys (in case they are not copied from NoSQL to SQL DB in hybrid mode.
@ -377,150 +437,24 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
return results;
}
private ListenableFuture<TbEntityDataSubCtx> handleHistoryCmd(TbEntityDataSubCtx ctx, EntityHistoryCmd historyCmd) {
List<ReadTsKvQuery> tsKvQueryList = historyCmd.getKeys().stream().map(key -> new BaseReadTsKvQuery(
key, historyCmd.getStartTs(), historyCmd.getEndTs(), historyCmd.getInterval(), getLimit(historyCmd.getLimit()), historyCmd.getAgg()
)).collect(Collectors.toList());
Map<EntityData, ListenableFuture<List<TsKvEntry>>> fetchResultMap = new HashMap<>();
ctx.getData().getData().forEach(entityData -> fetchResultMap.put(entityData,
tsService.findAll(ctx.getTenantId(), entityData.getEntityId(), tsKvQueryList)));
return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> {
fetchResultMap.forEach((entityData, future) -> {
Map<String, List<TsValue>> keyData = new LinkedHashMap<>();
historyCmd.getKeys().forEach(key -> keyData.put(key, new ArrayList<>()));
try {
List<TsKvEntry> entityTsData = future.get();
if (entityTsData != null) {
entityTsData.forEach(entry -> keyData.get(entry.getKey()).add(new TsValue(entry.getTs(), entry.getValueAsString())));
}
keyData.forEach((k, v) -> entityData.getTimeseries().put(k, v.toArray(new TsValue[v.size()])));
} catch (InterruptedException | ExecutionException e) {
log.warn("[{}][{}][{}] Failed to fetch historical data", ctx.getSessionId(), ctx.getCmdId(), entityData.getEntityId(), e);
wsService.sendWsMsg(ctx.getSessionId(),
new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to fetch historical data!"));
}
});
EntityDataUpdate update;
if (!ctx.isInitialDataSent()) {
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null);
ctx.setInitialDataSent(true);
} else {
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData());
}
wsService.sendWsMsg(ctx.getSessionId(), update);
return ctx;
}, wsCallBackExecutor);
}
@Override
public void cancelSubscription(String sessionId, EntityDataUnsubscribeCmd cmd) {
TbEntityDataSubCtx ctx = getSubCtx(sessionId, cmd.getCmdId());
cleanupAndCancel(getSubCtx(sessionId, cmd.getCmdId()));
}
// //TODO 3.1: replace null callbacks with callbacks from websocket service.
// @Override
// public void addSubscription(TbSubscription subscription) {
// EntityId entityId = subscription.getEntityId();
// // Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges;
// if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) {
// subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription);
// }
// pushSubscriptionToManagerService(subscription, true);
// registerSubscription(subscription);
// }
// private void pushSubscriptionToManagerService(TbSubscription subscription, boolean pushToLocalService) {
// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId());
// if (currentPartitions.contains(tpi)) {
// // Subscription is managed on the same server;
// if (pushToLocalService) {
// subscriptionManagerService.addSubscription(subscription, TbCallback.EMPTY);
// }
// } else {
// // Push to the queue;
// TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toNewSubscriptionProto(subscription);
// clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null);
// }
// }
@Override
public void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback) {
// TbSubscription subscription = subscriptionsBySessionId
// .getOrDefault(sessionId, Collections.emptyMap()).get(update.getSubscriptionId());
// if (subscription != null) {
// switch (subscription.getType()) {
// case TIMESERIES:
// TbTimeseriesSubscription tsSub = (TbTimeseriesSubscription) subscription;
// update.getLatestValues().forEach((key, value) -> tsSub.getKeyStates().put(key, value));
// break;
// case ATTRIBUTES:
// TbAttributeSubscription attrSub = (TbAttributeSubscription) subscription;
// update.getLatestValues().forEach((key, value) -> attrSub.getKeyStates().put(key, value));
// break;
// }
// wsService.sendWsMsg(sessionId, update);
// }
// callback.onSuccess();
private void cleanupAndCancel(TbEntityDataSubCtx ctx) {
if (ctx != null) {
ctx.cancelRefreshTask();
clearSubs(ctx);
}
}
// @Override
// public void cancelSubscription(String sessionId, int subscriptionId) {
// log.debug("[{}][{}] Going to remove subscription.", sessionId, subscriptionId);
// Map<Integer, TbSubscription> sessionSubscriptions = subscriptionsBySessionId.get(sessionId);
// if (sessionSubscriptions != null) {
// TbSubscription subscription = sessionSubscriptions.remove(subscriptionId);
// if (subscription != null) {
// if (sessionSubscriptions.isEmpty()) {
// subscriptionsBySessionId.remove(sessionId);
// }
// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, subscription.getTenantId(), subscription.getEntityId());
// if (currentPartitions.contains(tpi)) {
// // Subscription is managed on the same server;
// subscriptionManagerService.cancelSubscription(sessionId, subscriptionId, TbCallback.EMPTY);
// } else {
// // Push to the queue;
// TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toCloseSubscriptionProto(subscription);
// clusterService.pushMsgToCore(tpi, subscription.getEntityId().getId(), toCoreMsg, null);
// }
// } else {
// log.debug("[{}][{}] Subscription not found!", sessionId, subscriptionId);
// }
// } else {
// log.debug("[{}] No session subscriptions found!", sessionId);
// }
// }
@Override
public void cancelAllSessionSubscriptions(String sessionId) {
// Map<Integer, TbSubscription> subscriptions = subscriptionsBySessionId.get(sessionId);
// if (subscriptions != null) {
// Set<Integer> toRemove = new HashSet<>(subscriptions.keySet());
// toRemove.forEach(id -> cancelSubscription(sessionId, id));
// }
}
private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) {
EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId()));
Map<String, Long> keyStates;
if (subscription.isAllKeys()) {
keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L));
} else {
keyStates = subscription.getKeyStates().entrySet()
.stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
Map<Integer, TbEntityDataSubCtx> sessionSubs = subscriptionsBySessionId.remove(sessionId);
if (sessionSubs != null) {
sessionSubs.values().forEach(this::cleanupAndCancel);
}
return TbTimeseriesSubscription.builder()
.serviceId(subscription.getServiceId())
.sessionId(subscription.getSessionId())
.subscriptionId(subscription.getSubscriptionId())
.tenantId(subscription.getTenantId())
.entityId(entityView.getEntityId())
.startTime(entityView.getStartTimeMs())
.endTime(entityView.getEndTimeMs())
.allKeys(false)
.keyStates(keyStates).build();
}
private int getLimit(int limit) {

135
application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java
View File

@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.TsValue;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate;
import org.thingsboard.server.service.telemetry.cmd.v2.LatestValueCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.TimeSeriesCmd;
@ -39,9 +40,14 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@Data
@ -53,12 +59,14 @@ public class TbEntityDataSubCtx {
private final TelemetryWebSocketSessionRef sessionRef;
private final int cmdId;
private EntityDataQuery query;
private LatestValueCmd latestCmd;
private TimeSeriesCmd tsCmd;
private PageData<EntityData> data;
private boolean initialDataSent;
private List<TbSubscription> tbSubs;
private Map<Integer, EntityId> subToEntityIdMap;
private volatile ScheduledFuture<?> refreshTask;
private TimeSeriesCmd curTsCmd;
private LatestValueCmd latestValueCmd;
public TbEntityDataSubCtx(String serviceId, TelemetryWebSocketService wsService, TelemetryWebSocketSessionRef sessionRef, int cmdId) {
this.serviceId = serviceId;
@ -86,34 +94,43 @@ public class TbEntityDataSubCtx {
public List<TbSubscription> createSubscriptions(List<EntityKey> keys, boolean resultToLatestValues) {
this.subToEntityIdMap = new HashMap<>();
tbSubs = new ArrayList<>();
Map<EntityKeyType, List<EntityKey>> keysByType = new HashMap<>();
keys.forEach(key -> keysByType.computeIfAbsent(key.getType(), k -> new ArrayList<>()).add(key));
Map<EntityKeyType, List<EntityKey>> keysByType = getEntityKeyByTypeMap(keys);
for (EntityData entityData : data.getData()) {
keysByType.forEach((keysType, keysList) -> {
int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet();
subToEntityIdMap.put(subIdx, entityData.getEntityId());
switch (keysType) {
case TIME_SERIES:
tbSubs.add(createTsSub(entityData, subIdx, keysList, resultToLatestValues));
break;
case CLIENT_ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.CLIENT_SCOPE, keysList));
break;
case SHARED_ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.SHARED_SCOPE, keysList));
break;
case SERVER_ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.SERVER_SCOPE, keysList));
break;
case ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.ANY_SCOPE, keysList));
break;
}
});
addSubscription(entityData, keysByType, resultToLatestValues);
}
return tbSubs;
}
private Map<EntityKeyType, List<EntityKey>> getEntityKeyByTypeMap(List<EntityKey> keys) {
Map<EntityKeyType, List<EntityKey>> keysByType = new HashMap<>();
keys.forEach(key -> keysByType.computeIfAbsent(key.getType(), k -> new ArrayList<>()).add(key));
return keysByType;
}
private void addSubscription(EntityData entityData, Map<EntityKeyType, List<EntityKey>> keysByType, boolean resultToLatestValues) {
keysByType.forEach((keysType, keysList) -> {
int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet();
subToEntityIdMap.put(subIdx, entityData.getEntityId());
switch (keysType) {
case TIME_SERIES:
tbSubs.add(createTsSub(entityData, subIdx, keysList, resultToLatestValues));
break;
case CLIENT_ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.CLIENT_SCOPE, keysList));
break;
case SHARED_ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.SHARED_SCOPE, keysList));
break;
case SERVER_ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.SERVER_SCOPE, keysList));
break;
case ATTRIBUTE:
tbSubs.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.ANY_SCOPE, keysList));
break;
}
});
}
private TbSubscription createAttrSub(EntityData entityData, int subIdx, EntityKeyType keysType, TbAttributeSubscriptionScope scope, List<EntityKey> subKeys) {
Map<String, Long> keyStates = buildKeyStats(entityData, keysType, subKeys);
log.trace("[{}][{}][{}] Creating attributes subscription with keys: {}", serviceId, cmdId, subIdx, keyStates);
@ -275,4 +292,74 @@ public class TbEntityDataSubCtx {
return Collections.emptyList();
}
}
public void setRefreshTask(ScheduledFuture<?> task) {
this.refreshTask = task;
}
public void cancelRefreshTask() {
if (this.refreshTask != null) {
log.trace("[{}][{}] Canceling old refresh task", sessionRef.getSessionId(), cmdId);
this.refreshTask.cancel(true);
}
}
public Collection<Integer> update(PageData<EntityData> newData) {
Map<EntityId, EntityData> oldDataMap;
if (data != null && !data.getData().isEmpty()) {
oldDataMap = data.getData().stream().collect(Collectors.toMap(EntityData::getEntityId, Function.identity()));
} else {
oldDataMap = Collections.emptyMap();
}
Map<EntityId, EntityData> newDataMap = newData.getData().stream().collect(Collectors.toMap(EntityData::getEntityId, Function.identity()));
if (oldDataMap.size() == newDataMap.size() && oldDataMap.keySet().equals(newDataMap.keySet())) {
log.trace("[{}][{}] No updates to entity data found", sessionRef.getSessionId(), cmdId);
return Collections.emptyList();
} else {
this.data = newData;
List<Integer> subIdsToRemove = new ArrayList<>();
Set<EntityId> currentSubs = new HashSet<>();
subToEntityIdMap.forEach((subId, entityId) -> {
if (!newDataMap.containsKey(entityId)) {
subIdsToRemove.add(subId);
} else {
currentSubs.add(entityId);
}
});
log.trace("[{}][{}] Subscriptions that are invalid: {}", sessionRef.getSessionId(), cmdId, subIdsToRemove);
subIdsToRemove.forEach(subToEntityIdMap::remove);
List<EntityData> newSubsList = newDataMap.entrySet().stream().filter(entry -> !currentSubs.contains(entry.getKey())).map(Map.Entry::getValue).collect(Collectors.toList());
if (!newSubsList.isEmpty()) {
boolean resultToLatestValues;
List<EntityKey> keys = null;
if (curTsCmd != null) {
resultToLatestValues = false;
keys = curTsCmd.getKeys().stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).collect(Collectors.toList());
} else if (latestValueCmd != null) {
resultToLatestValues = true;
keys = latestValueCmd.getKeys();
} else {
resultToLatestValues = true;
}
if (keys != null && !keys.isEmpty()) {
Map<EntityKeyType, List<EntityKey>> keysByType = getEntityKeyByTypeMap(keys);
newSubsList.forEach(
entity -> {
log.trace("[{}][{}] Found new subscription for entity: {}", sessionRef.getSessionId(), cmdId, entity.getEntityId());
if (curTsCmd != null) {
addSubscription(entity, keysByType, resultToLatestValues);
}
}
);
}
}
wsService.sendWsMsg(sessionRef.getSessionId(), new EntityDataUpdate(cmdId, data, null));
return subIdsToRemove;
}
}
public void setCurrentCmd(EntityDataCmd cmd) {
curTsCmd = cmd.getTsCmd();
latestValueCmd = cmd.getLatestCmd();
}
}

9
application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubscriptionService.java
View File

@ -15,13 +15,9 @@
*/
package org.thingsboard.server.service.subscription;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd;
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUnsubscribeCmd;
import org.thingsboard.server.service.telemetry.sub.SubscriptionUpdate;
public interface TbEntityDataSubscriptionService {
@ -31,9 +27,4 @@ public interface TbEntityDataSubscriptionService {
void cancelAllSessionSubscriptions(String sessionId);
void onSubscriptionUpdate(String sessionId, SubscriptionUpdate update, TbCallback callback);
void onApplicationEvent(PartitionChangeEvent event);
void onApplicationEvent(ClusterTopologyChangeEvent event);
}

21
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java
View File

@ -697,15 +697,18 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
}
private void sendWsMsg(TelemetryWebSocketSessionRef sessionRef, int cmdId, Object update) {
executor.submit(() -> {
try {
msgEndpoint.send(sessionRef, cmdId, jsonMapper.writeValueAsString(update));
} catch (JsonProcessingException e) {
log.warn("[{}] Failed to encode reply: {}", sessionRef.getSessionId(), update, e);
} catch (IOException e) {
log.warn("[{}] Failed to send reply: {}", sessionRef.getSessionId(), update, e);
}
});
try {
String msg = jsonMapper.writeValueAsString(update);
executor.submit(() -> {
try {
msgEndpoint.send(sessionRef, cmdId, msg);
} catch (IOException e) {
log.warn("[{}] Failed to send reply: {}", sessionRef.getSessionId(), update, e);
}
});
} catch (JsonProcessingException e) {
log.warn("[{}] Failed to encode reply: {}", sessionRef.getSessionId(), update, e);
}
}

3
application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityHistoryCmd.java
View File

@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.kv.Aggregation;
import java.util.List;
@Data
public class EntityHistoryCmd {
public class EntityHistoryCmd implements GetTsCmd {
private List<String> keys;
private long startTs;
@ -29,5 +29,6 @@ public class EntityHistoryCmd {
private long interval;
private int limit;
private Aggregation agg;
private boolean fetchLatestPreviousPoint;
}

38
application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/GetTsCmd.java
View File

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.telemetry.cmd.v2;
import org.thingsboard.server.common.data.kv.Aggregation;
import java.util.List;
public interface GetTsCmd {
long getStartTs();
long getEndTs();
List<String> getKeys();
long getInterval();
int getLimit();
Aggregation getAgg();
boolean isFetchLatestPreviousPoint();
}

12
application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/TimeSeriesCmd.java
View File

@ -15,18 +15,26 @@
*/
package org.thingsboard.server.service.telemetry.cmd.v2;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import org.thingsboard.server.common.data.kv.Aggregation;
import java.util.List;
@Data
public class TimeSeriesCmd {
public class TimeSeriesCmd implements GetTsCmd {
private List<String> keys;
private long startTs;
private long timeWindow;
private long interval;
private int limit;
private String agg;
private Aggregation agg;
private boolean fetchLatestPreviousPoint;
@JsonIgnore
@Override
public long getEndTs() {
return startTs + timeWindow;
}
}

2
application/src/main/resources/thingsboard.yml
View File

@ -46,6 +46,8 @@ server:
max_subscriptions_per_regular_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_REGULAR_USER:0}"
max_subscriptions_per_public_user: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_PUBLIC_USER:0}"
max_updates_per_session: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_UPDATES_PER_SESSION:300:1,3000:60}"
dynamic_page_link_refresh_interval: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_INTERVAL_SEC:6}"
dynamic_page_link_refresh_pool_size: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_POOL_SIZE:1}"
rest:
limits:
tenant:

10
application/src/test/java/org/thingsboard/server/controller/BaseWebsocketApiTest.java
View File

@ -194,7 +194,7 @@ public class BaseWebsocketApiTest extends AbstractWebsocketTest {
TimeSeriesCmd tsCmd = new TimeSeriesCmd();
tsCmd.setKeys(Arrays.asList("temperature"));
tsCmd.setAgg(Aggregation.NONE.name());
tsCmd.setAgg(Aggregation.NONE);
tsCmd.setLimit(1000);
tsCmd.setStartTs(now - TimeUnit.HOURS.toMillis(1));
tsCmd.setTimeWindow(TimeUnit.HOURS.toMillis(1));
@ -561,16 +561,12 @@ public class BaseWebsocketApiTest extends AbstractWebsocketTest {
wsClient.registerWaitForUpdate();
AttributeKvEntry dataPoint1 = new BaseAttributeKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("serverAttributeKey", 42L));
List<AttributeKvEntry> tsData = Arrays.asList(dataPoint1);
sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, tsData);
Thread.sleep(100);
cmd = new EntityDataCmd(1, edq, null, latestCmd, null);
wrapper = new TelemetryPluginCmdsWrapper();
wrapper.setEntityDataCmds(Collections.singletonList(cmd));
sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, tsData);
msg = wsClient.waitForUpdate();
Assert.assertNotNull(msg);
update = mapper.readValue(msg, EntityDataUpdate.class);
Assert.assertEquals(1, update.getCmdId());
List<EntityData> eData = update.getUpdate();

10
common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java
View File

@ -23,27 +23,27 @@ public class BaseReadTsKvQuery extends BaseTsKvQuery implements ReadTsKvQuery {
private final long interval;
private final int limit;
private final Aggregation aggregation;
private final String orderBy;
private final String order;
public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation) {
this(key, startTs, endTs, interval, limit, aggregation, "DESC");
}
public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation,
String orderBy) {
String order) {
super(key, startTs, endTs);
this.interval = interval;
this.limit = limit;
this.aggregation = aggregation;
this.orderBy = orderBy;
this.order = order;
}
public BaseReadTsKvQuery(String key, long startTs, long endTs) {
this(key, startTs, endTs, endTs - startTs, 1, Aggregation.AVG, "DESC");
}
public BaseReadTsKvQuery(String key, long startTs, long endTs, int limit, String orderBy) {
this(key, startTs, endTs, endTs - startTs, limit, Aggregation.NONE, orderBy);
public BaseReadTsKvQuery(String key, long startTs, long endTs, int limit, String order) {
this(key, startTs, endTs, endTs - startTs, limit, Aggregation.NONE, order);
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java
View File

@ -23,6 +23,6 @@ public interface ReadTsKvQuery extends TsKvQuery {
Aggregation getAggregation();
String getOrderBy();
String getOrder();
}

6
common/data/src/main/java/org/thingsboard/server/common/data/query/EntityDataPageLink.java
View File

@ -27,13 +27,17 @@ public class EntityDataPageLink {
private int page;
private String textSearch;
private EntityDataSortOrder sortOrder;
private boolean dynamic = false;
public EntityDataPageLink() {
}
public EntityDataPageLink(int pageSize, int page, String textSearch, EntityDataSortOrder sortOrder) {
this(pageSize, page, textSearch, sortOrder, false);
}
@JsonIgnore
public EntityDataPageLink nextPageLink() {
return new EntityDataPageLink(this.pageSize, this.page+1, this.textSearch, this.sortOrder);
}
}

2
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
View File

@ -451,7 +451,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
List<String> searchPredicates = selectionMapping.stream().map(mapping -> {
String paramName = mapping.getValueAlias() + "_lowerSearchText";
ctx.addStringParameter(paramName, lowerSearchText);
return String.format("LOWER(%s) LIKE :%s", mapping.getValueAlias(), paramName);
return String.format("LOWER(%s) LIKE concat('%%', :%s, '%%')", mapping.getValueAlias(), paramName);
}
).collect(Collectors.toList());
return String.format(" WHERE %s", String.join(" or ", searchPredicates));

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
View File

@ -152,7 +152,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
query.getEndTs(),
PageRequest.of(0, query.getLimit(),
Sort.by(Sort.Direction.fromString(
query.getOrderBy()), "ts")));
query.getOrder()), "ts")));
tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey()));
return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities));
}

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java
View File

@ -110,7 +110,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
query.getEndTs(),
PageRequest.of(0, query.getLimit(),
Sort.by(Sort.Direction.fromString(
query.getOrderBy()), "ts")));
query.getOrder()), "ts")));
timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey));
return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities));
}

2
dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java
View File

@ -170,7 +170,7 @@ public class BaseTimeseriesService implements TimeseriesService {
} else {
endTs = query.getEndTs();
}
return new BaseReadTsKvQuery(query.getKey(), startTs, endTs, query.getInterval(), query.getLimit(), query.getAggregation(), query.getOrderBy());
return new BaseReadTsKvQuery(query.getKey(), startTs, endTs, query.getInterval(), query.getLimit(), query.getAggregation(), query.getOrder());
}).collect(Collectors.toList());
}

2
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
View File

@ -168,7 +168,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem
while (stepTs < query.getEndTs()) {
long startTs = stepTs;
long endTs = stepTs + step;
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrderBy());
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder());
futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs)));
stepTs = endTs;
}

2
dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java
View File

@ -40,7 +40,7 @@ public class TsKvQueryCursor extends QueryCursor {
public TsKvQueryCursor(String entityType, UUID entityId, ReadTsKvQuery baseQuery, List<Long> partitions) {
super(entityType, entityId, baseQuery, partitions);
this.orderBy = baseQuery.getOrderBy();
this.orderBy = baseQuery.getOrder();
this.partitionIndex = isDesc() ? partitions.size() - 1 : 0;
this.data = new ArrayList<>();
this.currentLimit = baseQuery.getLimit();

Write Preview
Loading…
Cancel
Save