From 32c18a882ab7a03531ca1851ac68a0b94531b41f Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 17 Nov 2021 15:43:35 +0200 Subject: [PATCH 01/15] Aggregation tsDao test: givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs --- ...stractChunkedAggregationTimeseriesDao.java | 2 +- ...ctChunkedAggregationTimeseriesDaoTest.java | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 9c2dce109b..4d023c6215 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -146,7 +146,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } - private ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); return Futures.transform(setFutures(entitiesFutures), entity -> { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java new file mode 100644 index 0000000000..b8e59daa17 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -0,0 +1,44 @@ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class AbstractChunkedAggregationTimeseriesDaoTest { + + AbstractChunkedAggregationTimeseriesDao tsDao; + + @Before + public void setUp() throws Exception { + tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); + ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); + willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { + ReadTsKvQuery query = new BaseReadTsKvQuery("temp", 0, 3000, 2000, 0, Aggregation.COUNT, "DESC"); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 0, 2000, 1000, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 2001, 3000, 2500, Aggregation.COUNT); + } +} \ No newline at end of file From aa88ec774e7f1d990641083589b3391dfb59620a Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 17 Nov 2021 16:43:40 +0200 Subject: [PATCH 02/15] add test with must use case --- ...ctChunkedAggregationTimeseriesDaoTest.java | 87 ++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index b8e59daa17..57706e61bd 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -23,6 +23,11 @@ import static org.mockito.Mockito.verify; public class AbstractChunkedAggregationTimeseriesDaoTest { + final int START_TS = 1; + final int LIMIT = 0; + final int END_TS = 3000; + final String TEMP = "temp"; + final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; @Before @@ -34,11 +39,87 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { - ReadTsKvQuery query = new BaseReadTsKvQuery("temp", 0, 3000, 2000, 0, Aggregation.COUNT, "DESC"); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 0, 2000, 1000, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 2001, 3000, 2500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 2000, 1000, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 2999, 1499, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsLongMax() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Long.MAX_VALUE, Long.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Long.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { + long intervalTs = 3; + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + for (long i = START_TS; i < END_TS; i += intervalTs) { + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, i, i + intervalTs, i, Aggregation.COUNT); + } } } \ No newline at end of file From d4d61d3e9b50ceafe709d55bb56da199124bb391 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 18 Nov 2021 16:19:06 +0200 Subject: [PATCH 03/15] refactoring code --- ...stractChunkedAggregationTimeseriesDao.java | 32 ++++++------- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 14 +++++- .../sqlts/BaseAbstractSqlTimeseriesDao.java | 24 ---------- .../dao/sqlts/SqlTimeseriesLatestDao.java | 1 + .../timescale/TimescaleTimeseriesDao.java | 10 ++-- .../CassandraBaseTimeseriesDao.java | 46 ++++++++----------- .../CassandraBaseTimeseriesLatestDao.java | 1 - ...ctChunkedAggregationTimeseriesDaoTest.java | 41 +++++++++++------ 8 files changed, 80 insertions(+), 89 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 4d023c6215..c7e9796eef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; @@ -115,24 +114,12 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(entityId, query); - } else { - long stepTs = query.getStartTs(); - List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + query.getInterval(); - long ts = startTs + (endTs - startTs) / 2; - futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); - stepTs = endTs; - } - return getTskvEntriesFuture(Futures.allAsList(futures)); - } + public long getTsForReadTsKvQuery(long startTs, long endTs) { + return startTs + (endTs - startTs) / 2; } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -146,6 +133,17 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } + @Override + public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { + return findAndAggregateAsync(entityId, + query.getKey(), + startTs, + endTs, + getTsForReadTsKvQuery(startTs, endTs), + aggregation); + + } + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index a5eee9c0f4..de0d3df4ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; +import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import javax.annotation.Nullable; import java.sql.Connection; @@ -35,6 +36,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.Objects; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -91,7 +93,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries .stream() .map(query -> findAllAsync(tenantId, entityId, query)) .collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { @@ -120,4 +122,14 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } + + @Override + public Executor getExecutor() { + return service; + } + + @Override + public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { + return interval; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 7e7f2ca00c..a6c6e5e61c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -15,33 +15,24 @@ */ package org.thingsboard.server.dao.sqlts; -import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; -import javax.annotation.Nullable; -import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; @Slf4j public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); - protected static final ReentrantLock tsCreationLock = new ReentrantLock(); - @Autowired protected TsKvDictionaryRepository dictionaryRepository; @@ -81,19 +72,4 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni return keyId; } - protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, service); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 3937b7c2c5..52d3ddf53b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -45,6 +45,7 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; +import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 435c9c5b70..15c211e6d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -34,7 +33,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; @@ -46,9 +44,6 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.sql.CallableStatement; -import java.sql.SQLException; -import java.sql.Types; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -165,6 +160,11 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements super.cleanup(systemTtl); } + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + return findAllAsyncWithLimit(entityId, query); + } + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 736b234eb9..3ebe14fdc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -37,7 +37,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.KvEntry; @@ -46,7 +45,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -61,6 +59,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -141,7 +140,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { @@ -266,29 +265,13 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(tenantId, entityId, query); - } else { - long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); - long stepTs = query.getStartTs(); - List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + step; - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder()); - futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); - stepTs = endTs; - } - ListenableFuture>> future = Futures.allAsList(futures); - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> input) { - return input == null ? Collections.emptyList() : input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList()); - } - }, readResultsProcessingExecutor); - } + public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { + return Math.max(interval, MIN_AGGREGATION_STEP_MS); + } + + @Override + public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { + return findAndAggregateAsync(tenantId, entityId, query, startTs, endTs); } @Override @@ -296,7 +279,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD //Cleanup by TTL is native for Cassandra } - private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); @@ -318,11 +302,17 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return resultFuture; } - private long toPartitionTs(long ts) { + @Override + public long toPartitionTs(long ts) { LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } + @Override + public Executor getExecutor() { + return readResultsProcessingExecutor; + } + private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SimpleListenableFuture> resultFuture) { if (cursor.isFull() || !cursor.hasNextPartition()) { resultFuture.set(cursor.getData()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 0b4ae71e85..c3b9d0cbb6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; -import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import javax.annotation.Nullable; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 57706e61bd..5fed9083e6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.sqlts; import com.google.common.util.concurrent.Futures; @@ -37,7 +52,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -47,7 +62,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -56,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -66,7 +81,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -75,7 +90,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -84,7 +99,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -93,25 +108,25 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); } - @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsLongMax() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Long.MAX_VALUE, Long.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + //@Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Long.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); From 729663e26d115ada79c9d44a9ccb80d41fd316a9 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 18 Nov 2021 16:21:55 +0200 Subject: [PATCH 04/15] refactoring code --- .../dao/sqlts/AggregationTimeseriesDao.java | 29 ------ .../timeseries/AggregationTimeseriesDao.java | 96 +++++++++++++++++++ ...ctChunkedAggregationTimeseriesDaoTest.java | 18 ++-- 3 files changed, 105 insertions(+), 38 deletions(-) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java deleted file mode 100644 index 4464428200..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts; - -import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import java.util.List; - -public interface AggregationTimeseriesDao { - - ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..bf6948b1ac --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -0,0 +1,96 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; + +public interface AggregationTimeseriesDao { + + default ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(tenantId, entityId, query); + } else { + long step = getIntervalGreaterOrEqualsMinAggregationStep(query.getInterval()); + long stepTs = query.getStartTs(); + List>> futures = findIntervals(tenantId, entityId, query, step, stepTs); + return getTskvEntriesFuture(Futures.allAsList(futures)); + } + } + + default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long step, long stepTs) { + List>> futures = new ArrayList<>(); + while (stepTs < query.getEndTs()) { + long startTs = stepTs; + long endTs = stepTs + step; + long ts = getTsForReadTsKvQuery(startTs, endTs); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); + futures.add(aggregateTsKvEntry); + stepTs = endTs; + } + return futures; + } + + default long getTsForReadTsKvQuery(long startTs, long endTs) { + return endTs - startTs; + } + + long getIntervalGreaterOrEqualsMinAggregationStep(long interval); + + default ListenableFuture> getTskvEntriesFuture(ListenableFuture>> allAsList) { + return Futures.transform(allAsList, new Function<>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, getExecutor()); + } + + Executor getExecutor(); + + default ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery key, long startTs, long endTs, Aggregation aggregation) { + return Futures.immediateFuture(null); + } + + ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); + + default long toPartitionTs(long ts) { + return ts; + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 5fed9083e6..43c2467885 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -52,7 +52,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -62,7 +62,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -71,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -81,7 +81,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -90,7 +90,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -99,7 +99,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -108,7 +108,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -117,7 +117,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -126,7 +126,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); From 76c7b34c3b242c8c98427e8a093c4c8df3d7e13c Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 19 Nov 2021 14:03:30 +0200 Subject: [PATCH 05/15] * refactoring test and code * fix bug with interval in findAllAsync() --- .../timeseries/AggregationTimeseriesDao.java | 17 ++--- ...ctChunkedAggregationTimeseriesDaoTest.java | 69 ++++++++++++------- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java index bf6948b1ac..a99ad053b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -38,23 +38,24 @@ public interface AggregationTimeseriesDao { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { - long step = getIntervalGreaterOrEqualsMinAggregationStep(query.getInterval()); - long stepTs = query.getStartTs(); - List>> futures = findIntervals(tenantId, entityId, query, step, stepTs); + List>> futures = findIntervals(tenantId, entityId, query); return getTskvEntriesFuture(Futures.allAsList(futures)); } } - default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long step, long stepTs) { + default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + step; + long endPeriod = query.getEndTs(); + long startPeriod = query.getStartTs(); + long step = query.getInterval(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); long ts = getTsForReadTsKvQuery(startTs, endTs); ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); futures.add(aggregateTsKvEntry); - stepTs = endTs; + startPeriod = endTs; } return futures; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 43c2467885..b8a2328fe2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -39,91 +39,109 @@ import static org.mockito.Mockito.verify; public class AbstractChunkedAggregationTimeseriesDaoTest { final int START_TS = 1; - final int LIMIT = 0; + final int LIMIT = 1; final int END_TS = 3000; final String TEMP = "temp"; final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; + //SOME PRESENT: When we give data with period l-r, program return data in interval [l;r) + @Before public void setUp() throws Exception { tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(null)).given(tsDao).getTskvEntriesFuture(any()); + willCallRealMethod().given(tsDao).findAllAsync(any(), any(), any(ReadTsKvQuery.class)); + willCallRealMethod().given(tsDao).findIntervals(any(), any(), any(ReadTsKvQuery.class)); + willCallRealMethod().given(tsDao).getTsForReadTsKvQuery(anyLong(), anyLong()); + willCallRealMethod().given(tsDao).toPartitionTs(anyLong()); + willCallRealMethod().given(tsDao).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 2000, 1000, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, 2001, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, 2001, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 2999, 1499, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, END_TS, END_TS + 1, Aggregation.COUNT); + } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, 0, 0 + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query, START_TS, START_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test @@ -132,9 +150,10 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1000)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - for (long i = START_TS; i < END_TS; i += intervalTs) { - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, i, i + intervalTs, i, Aggregation.COUNT); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + for (long i = START_TS; i <= END_TS; i += intervalTs) { + ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + intervalTs, i + (i + intervalTs - i) / 2, LIMIT, Aggregation.COUNT, DESC); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, querySub, i, i + intervalTs, Aggregation.COUNT); } } } \ No newline at end of file From 32fb91bce560015d8d02fd6a8fafe5b34a1300b1 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 19 Nov 2021 15:09:52 +0200 Subject: [PATCH 06/15] fixed an issue with grammar mistake in tests name --- ...actChunkedAggregationTimeseriesDaoTest.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index b8a2328fe2..01f0ba73fe 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -61,7 +61,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); @@ -73,7 +73,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -83,7 +83,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); @@ -96,7 +96,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -106,7 +106,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -116,7 +116,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -125,7 +125,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -135,7 +135,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -145,7 +145,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); From df2758e8e4bdac03687a95a2f4d420e2198ce7ae Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 22 Nov 2021 12:53:30 +0200 Subject: [PATCH 07/15] refactoring test and start add test in service level --- .../timeseries/BaseTimeseriesServiceTest.java | 26 ++++ ...ctChunkedAggregationTimeseriesDaoTest.java | 117 ++++++++---------- 2 files changed, 81 insertions(+), 62 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index b7872fc239..0bfcff6832 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -164,6 +164,32 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } +// +// @Test +// public void testFindByQueryOneMilisecondPeriod() throws Exception { +// DeviceId deviceId = new DeviceId(Uuids.timeBased()); +// +// saveEntries(deviceId, TS - 3); +// saveEntries(deviceId, TS - 2); +// saveEntries(deviceId, TS - 1); +// +// List queries = new ArrayList<>(); +// queries.add(new BaseReadTsKvQuery(STRING_KEY, TS, TS, 0, 1000, Aggregation.NONE, "ASC")); +// +// List entries = tsService.findAll(tenantId, deviceId, queries).get(); +// Assert.assertEquals(3, entries.size()); +// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); +// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); +// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); +// +// EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); +// +// entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); +// Assert.assertEquals(3, entries.size()); +// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); +// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); +// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); +// } @Test public void testFindByQueryDescOrder() throws Exception { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 01f0ba73fe..ae3d27fdf5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,28 +19,28 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.BDDMockito.willCallRealMethod; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.thingsboard.server.common.data.id.TenantId.SYS_TENANT_ID; +import static org.thingsboard.server.common.data.kv.Aggregation.COUNT; public class AbstractChunkedAggregationTimeseriesDaoTest { - final int START_TS = 1; final int LIMIT = 1; - final int END_TS = 3000; final String TEMP = "temp"; final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; @@ -49,111 +49,104 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Before public void setUp() throws Exception { - tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); + tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - willReturn(Futures.immediateFuture(null)).given(tsDao).getTskvEntriesFuture(any()); - willCallRealMethod().given(tsDao).findAllAsync(any(), any(), any(ReadTsKvQuery.class)); - willCallRealMethod().given(tsDao).findIntervals(any(), any(), any(ReadTsKvQuery.class)); - willCallRealMethod().given(tsDao).getTsForReadTsKvQuery(anyLong(), anyLong()); - willCallRealMethod().given(tsDao).toPartitionTs(anyLong()); - willCallRealMethod().given(tsDao).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any()); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); - - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, 2001, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, 2001, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 2001, 3000 + 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000 + 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2998, 1500, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2999, 3000, 3000, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, END_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 3000, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, 0, 0 + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 0, 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query, START_TS, START_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, query, 1, 2, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() { - long intervalTs = 3; - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - for (long i = START_TS; i <= END_TS; i += intervalTs) { - ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + intervalTs, i + (i + intervalTs - i) / 2, LIMIT, Aggregation.COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, querySub, i, i + intervalTs, Aggregation.COUNT); + for (long i = 1; i <= 3000; i += 3) { + ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, querySub, i, i + 3, COUNT); } } -} \ No newline at end of file +} From 5468d27db8c17a43bf81af664856e9cdb8802415 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 22 Nov 2021 17:38:35 +0200 Subject: [PATCH 08/15] add test for Timeseries Service, where we check findAll() with Aggregation.COUNT. --- .../timeseries/BaseTimeseriesServiceTest.java | 158 +++++++++++++++--- 1 file changed, 132 insertions(+), 26 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 0bfcff6832..672c17614c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -164,32 +164,138 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } -// -// @Test -// public void testFindByQueryOneMilisecondPeriod() throws Exception { -// DeviceId deviceId = new DeviceId(Uuids.timeBased()); -// -// saveEntries(deviceId, TS - 3); -// saveEntries(deviceId, TS - 2); -// saveEntries(deviceId, TS - 1); -// -// List queries = new ArrayList<>(); -// queries.add(new BaseReadTsKvQuery(STRING_KEY, TS, TS, 0, 1000, Aggregation.NONE, "ASC")); -// -// List entries = tsService.findAll(tenantId, deviceId, queries).get(); -// Assert.assertEquals(3, entries.size()); -// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); -// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); -// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); -// -// EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); -// -// entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); -// Assert.assertEquals(3, entries.size()); -// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); -// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); -// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); -// } + + @Test + public void testFindByQueryOneMilisecondPeriod() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS); + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 5 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 5, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 8L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS - 2; i<= TS + 12L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS - 2L; i<= TS + 12L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + } @Test public void testFindByQueryDescOrder() throws Exception { From 38f64a7ac36286c8d20731c7d19b8823ea7c183d Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 26 Nov 2021 16:45:41 +0200 Subject: [PATCH 09/15] refactoring test after code review --- .../timeseries/BaseTimeseriesServiceTest.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 672c17614c..2c5d8b9fea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -166,12 +166,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { } @Test - public void testFindByQueryOneMilisecondPeriod() throws Exception { + public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); saveEntries(deviceId, TS); + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); @@ -187,12 +188,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 10L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); @@ -208,12 +210,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 10L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -231,12 +234,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 8L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -258,8 +262,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { saveEntries(deviceId, i); } - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -281,8 +284,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { saveEntries(deviceId, i); } - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); From cc67e93f972e1e1147fb4f4df1f93b6e77138670 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 29 Nov 2021 11:28:04 +0200 Subject: [PATCH 10/15] change tests cases for more understandable --- .../timeseries/BaseTimeseriesServiceTest.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 2c5d8b9fea..b6ca2d639c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -168,9 +168,9 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); + saveEntries(deviceId, TS - 1L); saveEntries(deviceId, TS); - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 1L); List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); @@ -188,115 +188,115 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 5 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 5, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 8L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 80L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 80L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i=TS - 2; i<= TS + 12L; i++) { + for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i=TS - 2L; i<= TS + 12L; i++) { + for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10L) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test From b54aa55c1ff6c82575b1604257fd1a185c0c7403 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 17 Jan 2022 12:49:36 +0200 Subject: [PATCH 11/15] update tests --- ...ctChunkedAggregationTimeseriesDaoTest.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index ae3d27fdf5..95cd2280fe 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,6 +19,8 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -59,7 +61,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); @@ -69,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -79,8 +81,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2998, 1500, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2999, 3000, 3000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -92,7 +94,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -102,7 +104,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -112,10 +114,11 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, query, 1, 2, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuery, 1, 2, COUNT); } @Test @@ -131,7 +134,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); From d02c7ad7bf0346098bae1786142c053d1f75dc4c Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 17 Jan 2022 18:41:34 +0200 Subject: [PATCH 12/15] fix issue with license --- .../server/dao/timeseries/AggregationTimeseriesDao.java | 2 +- .../dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java index a99ad053b6..b8e42e3165 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 95cd2280fe..68fc8feccc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 51071a7413cf8a3d576af8627806be015869d363 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 19 Jan 2022 18:06:53 +0200 Subject: [PATCH 13/15] remove unnecessary code and add some clean code --- .../server/dao/sqlts/SqlTimeseriesLatestDao.java | 1 - .../server/dao/sqlts/timescale/TimescaleTimeseriesDao.java | 7 ++++++- .../dao/service/timeseries/BaseTimeseriesServiceTest.java | 2 +- .../sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java | 4 ---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 6f58f318a1..7808f0e1bd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -46,7 +46,6 @@ import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index e15ee31b35..15875eb275 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -44,7 +44,12 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Function; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 7677654ec7..04c3425169 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -217,7 +217,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 50 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 68fc8feccc..cab869a647 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,8 +19,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -47,8 +45,6 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; - //SOME PRESENT: When we give data with period l-r, program return data in interval [l;r) - @Before public void setUp() throws Exception { tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); From 7d7701723cc753a38f5e95a700b66b1fc2414f4e Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 22 Jun 2022 23:09:58 +0300 Subject: [PATCH 14/15] improve logic --- ...stractChunkedAggregationTimeseriesDao.java | 36 ++++--- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 12 --- .../dao/sqlts/AggregationTimeseriesDao.java | 29 ++++++ .../sqlts/BaseAbstractSqlTimeseriesDao.java | 23 +++++ .../dao/sqlts/SqlTimeseriesLatestDao.java | 1 - .../timescale/TimescaleTimeseriesDao.java | 5 - .../timeseries/AggregationTimeseriesDao.java | 97 ------------------- .../CassandraBaseTimeseriesDao.java | 46 +++++---- .../CassandraBaseTimeseriesLatestDao.java | 1 + ...ctChunkedAggregationTimeseriesDaoTest.java | 45 +++++---- 10 files changed, 128 insertions(+), 167 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 57b458a663..d68271798a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -26,6 +26,7 @@ import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -118,12 +119,28 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public long getTsForReadTsKvQuery(long startTs, long endTs) { - return startTs + (endTs - startTs) / 2; + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(entityId, query); + } else { + List>> futures = new ArrayList<>(); + long endPeriod = query.getEndTs(); + long startPeriod = query.getStartTs(); + long step = query.getInterval(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); + long ts = startTs + (endTs - startTs) / 2; + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, subQuery.getKey(), startTs, endTs, ts, query.getAggregation()); + futures.add(aggregateTsKvEntry); + startPeriod = endTs; + } + return getTskvEntriesFuture(Futures.allAsList(futures)); + } } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -136,17 +153,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } - @Override - public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { - return findAndAggregateAsync(entityId, - query.getKey(), - startTs, - endTs, - getTsForReadTsKvQuery(startTs, endTs), - aggregation); - - } - ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index c32123cb94..aa0d9b1c47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; -import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import javax.annotation.Nullable; import java.sql.Connection; @@ -36,7 +35,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.Objects; -import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -122,14 +120,4 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } - - @Override - public Executor getExecutor() { - return service; - } - - @Override - public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { - return interval; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..31270bacc7 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface AggregationTimeseriesDao { + + ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 83a5d11623..dd9d15b8b1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -15,18 +15,25 @@ */ package org.thingsboard.server.dao.sqlts; +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import javax.annotation.Nullable; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; @Slf4j public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { @@ -72,4 +79,20 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni return keyId; } + protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { + return Futures.transform(future, new Function>, List>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, service); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 7808f0e1bd..db30adead2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -45,7 +45,6 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 15875eb275..9c03a9e3dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -165,11 +165,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements super.cleanup(systemTtl); } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - return findAllAsyncWithLimit(entityId, query); - } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java deleted file mode 100644 index b8e42e3165..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.timeseries; - -import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -public interface AggregationTimeseriesDao { - - default ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(tenantId, entityId, query); - } else { - List>> futures = findIntervals(tenantId, entityId, query); - return getTskvEntriesFuture(Futures.allAsList(futures)); - } - } - - default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - List>> futures = new ArrayList<>(); - long endPeriod = query.getEndTs(); - long startPeriod = query.getStartTs(); - long step = query.getInterval(); - while (startPeriod <= endPeriod) { - long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod + 1); - long ts = getTsForReadTsKvQuery(startTs, endTs); - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); - ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); - futures.add(aggregateTsKvEntry); - startPeriod = endTs; - } - return futures; - } - - default long getTsForReadTsKvQuery(long startTs, long endTs) { - return endTs - startTs; - } - - long getIntervalGreaterOrEqualsMinAggregationStep(long interval); - - default ListenableFuture> getTskvEntriesFuture(ListenableFuture>> allAsList) { - return Futures.transform(allAsList, new Function<>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, getExecutor()); - } - - Executor getExecutor(); - - default ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery key, long startTs, long endTs, Aggregation aggregation) { - return Futures.immediateFuture(null); - } - - ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); - - default long toPartitionTs(long ts) { - return ts; - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 8b58e547c9..bb3930b99c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -37,6 +37,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.KvEntry; @@ -45,6 +46,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -59,7 +61,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -265,13 +266,31 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { - return Math.max(interval, MIN_AGGREGATION_STEP_MS); - } - - @Override - public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { - return findAndAggregateAsync(tenantId, entityId, query, startTs, endTs); + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(tenantId, entityId, query); + } else { + long startPeriod = query.getStartTs(); + long endPeriod = query.getEndTs(); + long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); + List>> futures = new ArrayList<>(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); + long ts = endTs - startTs; + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); + startPeriod = endTs; + } + ListenableFuture>> future = Futures.allAsList(futures); + return Futures.transform(future, new Function<>() { + @Nullable + @Override + public List apply(@Nullable List> input) { + return input == null ? Collections.emptyList() : input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList()); + } + }, readResultsProcessingExecutor); + } } @Override @@ -279,8 +298,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD //Cleanup by TTL is native for Cassandra } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); @@ -302,17 +320,11 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return resultFuture; } - @Override - public long toPartitionTs(long ts) { + private long toPartitionTs(long ts) { LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } - @Override - public Executor getExecutor() { - return readResultsProcessingExecutor; - } - private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SimpleListenableFuture> resultFuture) { if (cursor.isFull() || !cursor.hasNextPartition()) { resultFuture.set(cursor.getData()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index e7099db3d2..124af03a1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index cab869a647..db216f7f6e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -59,9 +59,9 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 2001, 3000 + 1, COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT); } @Test @@ -70,8 +70,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000 + 1, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -81,9 +81,9 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000, COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 3000, 3001, COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT); } @@ -93,8 +93,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -103,8 +103,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 0, 1, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 0, 1, getTsForReadTsKvQuery(0, 1), COUNT); } @Test @@ -113,8 +113,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuery, 1, 2, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuery.getKey(), 1, 2, getTsForReadTsKvQuery(1, 2), COUNT); } @Test @@ -123,8 +123,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), Integer.MAX_VALUE, 1L + Integer.MAX_VALUE, getTsForReadTsKvQuery(Integer.MAX_VALUE, 1L + Integer.MAX_VALUE), COUNT); } @Test @@ -133,8 +133,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -142,10 +142,15 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); for (long i = 1; i <= 3000; i += 3) { ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, querySub, i, i + 3, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT); } } + + long getTsForReadTsKvQuery(long startTs, long endTs) { + return startTs + (endTs - startTs) / 2L; + } + } From 4f8806dd8bc98995f7c1c97d0a415486d20fa5ec Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 23 Jun 2022 13:47:28 +0300 Subject: [PATCH 15/15] fix tests in nosql --- .../timeseries/BaseTimeseriesServiceTest.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 04c3425169..b38afe77f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -230,92 +230,92 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); - for (long i = TS; i <= TS + 100L; i += 10L) { + for (long i = TS; i <= TS + 100000L; i += 10000L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS + 100L + 1L); + saveEntries(deviceId, TS + 100000L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); - for (long i = TS; i <= TS + 80L; i += 10L) { + for (long i = TS; i <= TS + 80000L; i += 10000L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS + 80L + 1L); + saveEntries(deviceId, TS + 80000L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10) { + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10L) { + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test