From 32c18a882ab7a03531ca1851ac68a0b94531b41f Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 17 Nov 2021 15:43:35 +0200 Subject: [PATCH 01/24] Aggregation tsDao test: givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs --- ...stractChunkedAggregationTimeseriesDao.java | 2 +- ...ctChunkedAggregationTimeseriesDaoTest.java | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 9c2dce109b..4d023c6215 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -146,7 +146,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } - private ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); return Futures.transform(setFutures(entitiesFutures), entity -> { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java new file mode 100644 index 0000000000..b8e59daa17 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -0,0 +1,44 @@ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class AbstractChunkedAggregationTimeseriesDaoTest { + + AbstractChunkedAggregationTimeseriesDao tsDao; + + @Before + public void setUp() throws Exception { + tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); + ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); + willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { + ReadTsKvQuery query = new BaseReadTsKvQuery("temp", 0, 3000, 2000, 0, Aggregation.COUNT, "DESC"); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 0, 2000, 1000, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 2001, 3000, 2500, Aggregation.COUNT); + } +} \ No newline at end of file From aa88ec774e7f1d990641083589b3391dfb59620a Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 17 Nov 2021 16:43:40 +0200 Subject: [PATCH 02/24] add test with must use case --- ...ctChunkedAggregationTimeseriesDaoTest.java | 87 ++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index b8e59daa17..57706e61bd 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -23,6 +23,11 @@ import static org.mockito.Mockito.verify; public class AbstractChunkedAggregationTimeseriesDaoTest { + final int START_TS = 1; + final int LIMIT = 0; + final int END_TS = 3000; + final String TEMP = "temp"; + final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; @Before @@ -34,11 +39,87 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { - ReadTsKvQuery query = new BaseReadTsKvQuery("temp", 0, 3000, 2000, 0, Aggregation.COUNT, "DESC"); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 0, 2000, 1000, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 2001, 3000, 2500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 2000, 1000, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 2999, 1499, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsLongMax() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Long.MAX_VALUE, Long.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Long.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { + long intervalTs = 3; + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + for (long i = START_TS; i < END_TS; i += intervalTs) { + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, i, i + intervalTs, i, Aggregation.COUNT); + } } } \ No newline at end of file From d4d61d3e9b50ceafe709d55bb56da199124bb391 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 18 Nov 2021 16:19:06 +0200 Subject: [PATCH 03/24] refactoring code --- ...stractChunkedAggregationTimeseriesDao.java | 32 ++++++------- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 14 +++++- .../sqlts/BaseAbstractSqlTimeseriesDao.java | 24 ---------- .../dao/sqlts/SqlTimeseriesLatestDao.java | 1 + .../timescale/TimescaleTimeseriesDao.java | 10 ++-- .../CassandraBaseTimeseriesDao.java | 46 ++++++++----------- .../CassandraBaseTimeseriesLatestDao.java | 1 - ...ctChunkedAggregationTimeseriesDaoTest.java | 41 +++++++++++------ 8 files changed, 80 insertions(+), 89 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 4d023c6215..c7e9796eef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; @@ -115,24 +114,12 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(entityId, query); - } else { - long stepTs = query.getStartTs(); - List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + query.getInterval(); - long ts = startTs + (endTs - startTs) / 2; - futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); - stepTs = endTs; - } - return getTskvEntriesFuture(Futures.allAsList(futures)); - } + public long getTsForReadTsKvQuery(long startTs, long endTs) { + return startTs + (endTs - startTs) / 2; } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -146,6 +133,17 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } + @Override + public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { + return findAndAggregateAsync(entityId, + query.getKey(), + startTs, + endTs, + getTsForReadTsKvQuery(startTs, endTs), + aggregation); + + } + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index a5eee9c0f4..de0d3df4ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; +import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import javax.annotation.Nullable; import java.sql.Connection; @@ -35,6 +36,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.Objects; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -91,7 +93,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries .stream() .map(query -> findAllAsync(tenantId, entityId, query)) .collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { @@ -120,4 +122,14 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } + + @Override + public Executor getExecutor() { + return service; + } + + @Override + public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { + return interval; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 7e7f2ca00c..a6c6e5e61c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -15,33 +15,24 @@ */ package org.thingsboard.server.dao.sqlts; -import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; -import javax.annotation.Nullable; -import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; @Slf4j public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); - protected static final ReentrantLock tsCreationLock = new ReentrantLock(); - @Autowired protected TsKvDictionaryRepository dictionaryRepository; @@ -81,19 +72,4 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni return keyId; } - protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, service); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 3937b7c2c5..52d3ddf53b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -45,6 +45,7 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; +import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 435c9c5b70..15c211e6d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -34,7 +33,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; @@ -46,9 +44,6 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.sql.CallableStatement; -import java.sql.SQLException; -import java.sql.Types; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -165,6 +160,11 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements super.cleanup(systemTtl); } + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + return findAllAsyncWithLimit(entityId, query); + } + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 736b234eb9..3ebe14fdc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -37,7 +37,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.KvEntry; @@ -46,7 +45,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -61,6 +59,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -141,7 +140,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { @@ -266,29 +265,13 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(tenantId, entityId, query); - } else { - long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); - long stepTs = query.getStartTs(); - List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + step; - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder()); - futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); - stepTs = endTs; - } - ListenableFuture>> future = Futures.allAsList(futures); - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> input) { - return input == null ? Collections.emptyList() : input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList()); - } - }, readResultsProcessingExecutor); - } + public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { + return Math.max(interval, MIN_AGGREGATION_STEP_MS); + } + + @Override + public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { + return findAndAggregateAsync(tenantId, entityId, query, startTs, endTs); } @Override @@ -296,7 +279,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD //Cleanup by TTL is native for Cassandra } - private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); @@ -318,11 +302,17 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return resultFuture; } - private long toPartitionTs(long ts) { + @Override + public long toPartitionTs(long ts) { LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } + @Override + public Executor getExecutor() { + return readResultsProcessingExecutor; + } + private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SimpleListenableFuture> resultFuture) { if (cursor.isFull() || !cursor.hasNextPartition()) { resultFuture.set(cursor.getData()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 0b4ae71e85..c3b9d0cbb6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; -import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import javax.annotation.Nullable; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 57706e61bd..5fed9083e6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.sqlts; import com.google.common.util.concurrent.Futures; @@ -37,7 +52,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -47,7 +62,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -56,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -66,7 +81,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -75,7 +90,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -84,7 +99,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -93,25 +108,25 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); } - @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsLongMax() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Long.MAX_VALUE, Long.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + //@Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Long.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); From 729663e26d115ada79c9d44a9ccb80d41fd316a9 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 18 Nov 2021 16:21:55 +0200 Subject: [PATCH 04/24] refactoring code --- .../dao/sqlts/AggregationTimeseriesDao.java | 29 ------ .../timeseries/AggregationTimeseriesDao.java | 96 +++++++++++++++++++ ...ctChunkedAggregationTimeseriesDaoTest.java | 18 ++-- 3 files changed, 105 insertions(+), 38 deletions(-) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java deleted file mode 100644 index 4464428200..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts; - -import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import java.util.List; - -public interface AggregationTimeseriesDao { - - ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..bf6948b1ac --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -0,0 +1,96 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; + +public interface AggregationTimeseriesDao { + + default ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(tenantId, entityId, query); + } else { + long step = getIntervalGreaterOrEqualsMinAggregationStep(query.getInterval()); + long stepTs = query.getStartTs(); + List>> futures = findIntervals(tenantId, entityId, query, step, stepTs); + return getTskvEntriesFuture(Futures.allAsList(futures)); + } + } + + default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long step, long stepTs) { + List>> futures = new ArrayList<>(); + while (stepTs < query.getEndTs()) { + long startTs = stepTs; + long endTs = stepTs + step; + long ts = getTsForReadTsKvQuery(startTs, endTs); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); + futures.add(aggregateTsKvEntry); + stepTs = endTs; + } + return futures; + } + + default long getTsForReadTsKvQuery(long startTs, long endTs) { + return endTs - startTs; + } + + long getIntervalGreaterOrEqualsMinAggregationStep(long interval); + + default ListenableFuture> getTskvEntriesFuture(ListenableFuture>> allAsList) { + return Futures.transform(allAsList, new Function<>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, getExecutor()); + } + + Executor getExecutor(); + + default ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery key, long startTs, long endTs, Aggregation aggregation) { + return Futures.immediateFuture(null); + } + + ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); + + default long toPartitionTs(long ts) { + return ts; + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 5fed9083e6..43c2467885 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -52,7 +52,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -62,7 +62,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -71,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -81,7 +81,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -90,7 +90,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -99,7 +99,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -108,7 +108,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -117,7 +117,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -126,7 +126,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); From 76c7b34c3b242c8c98427e8a093c4c8df3d7e13c Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 19 Nov 2021 14:03:30 +0200 Subject: [PATCH 05/24] * refactoring test and code * fix bug with interval in findAllAsync() --- .../timeseries/AggregationTimeseriesDao.java | 17 ++--- ...ctChunkedAggregationTimeseriesDaoTest.java | 69 ++++++++++++------- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java index bf6948b1ac..a99ad053b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -38,23 +38,24 @@ public interface AggregationTimeseriesDao { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { - long step = getIntervalGreaterOrEqualsMinAggregationStep(query.getInterval()); - long stepTs = query.getStartTs(); - List>> futures = findIntervals(tenantId, entityId, query, step, stepTs); + List>> futures = findIntervals(tenantId, entityId, query); return getTskvEntriesFuture(Futures.allAsList(futures)); } } - default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long step, long stepTs) { + default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + step; + long endPeriod = query.getEndTs(); + long startPeriod = query.getStartTs(); + long step = query.getInterval(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); long ts = getTsForReadTsKvQuery(startTs, endTs); ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); futures.add(aggregateTsKvEntry); - stepTs = endTs; + startPeriod = endTs; } return futures; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 43c2467885..b8a2328fe2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -39,91 +39,109 @@ import static org.mockito.Mockito.verify; public class AbstractChunkedAggregationTimeseriesDaoTest { final int START_TS = 1; - final int LIMIT = 0; + final int LIMIT = 1; final int END_TS = 3000; final String TEMP = "temp"; final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; + //SOME PRESENT: When we give data with period l-r, program return data in interval [l;r) + @Before public void setUp() throws Exception { tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(null)).given(tsDao).getTskvEntriesFuture(any()); + willCallRealMethod().given(tsDao).findAllAsync(any(), any(), any(ReadTsKvQuery.class)); + willCallRealMethod().given(tsDao).findIntervals(any(), any(), any(ReadTsKvQuery.class)); + willCallRealMethod().given(tsDao).getTsForReadTsKvQuery(anyLong(), anyLong()); + willCallRealMethod().given(tsDao).toPartitionTs(anyLong()); + willCallRealMethod().given(tsDao).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 2000, 1000, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, 2001, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, 2001, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 2999, 1499, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, END_TS, END_TS + 1, Aggregation.COUNT); + } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, 0, 0 + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query, START_TS, START_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test @@ -132,9 +150,10 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1000)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - for (long i = START_TS; i < END_TS; i += intervalTs) { - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, i, i + intervalTs, i, Aggregation.COUNT); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + for (long i = START_TS; i <= END_TS; i += intervalTs) { + ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + intervalTs, i + (i + intervalTs - i) / 2, LIMIT, Aggregation.COUNT, DESC); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, querySub, i, i + intervalTs, Aggregation.COUNT); } } } \ No newline at end of file From 32fb91bce560015d8d02fd6a8fafe5b34a1300b1 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 19 Nov 2021 15:09:52 +0200 Subject: [PATCH 06/24] fixed an issue with grammar mistake in tests name --- ...actChunkedAggregationTimeseriesDaoTest.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index b8a2328fe2..01f0ba73fe 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -61,7 +61,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); @@ -73,7 +73,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -83,7 +83,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); @@ -96,7 +96,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -106,7 +106,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -116,7 +116,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -125,7 +125,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -135,7 +135,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -145,7 +145,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); From df2758e8e4bdac03687a95a2f4d420e2198ce7ae Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 22 Nov 2021 12:53:30 +0200 Subject: [PATCH 07/24] refactoring test and start add test in service level --- .../timeseries/BaseTimeseriesServiceTest.java | 26 ++++ ...ctChunkedAggregationTimeseriesDaoTest.java | 117 ++++++++---------- 2 files changed, 81 insertions(+), 62 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index b7872fc239..0bfcff6832 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -164,6 +164,32 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } +// +// @Test +// public void testFindByQueryOneMilisecondPeriod() throws Exception { +// DeviceId deviceId = new DeviceId(Uuids.timeBased()); +// +// saveEntries(deviceId, TS - 3); +// saveEntries(deviceId, TS - 2); +// saveEntries(deviceId, TS - 1); +// +// List queries = new ArrayList<>(); +// queries.add(new BaseReadTsKvQuery(STRING_KEY, TS, TS, 0, 1000, Aggregation.NONE, "ASC")); +// +// List entries = tsService.findAll(tenantId, deviceId, queries).get(); +// Assert.assertEquals(3, entries.size()); +// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); +// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); +// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); +// +// EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); +// +// entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); +// Assert.assertEquals(3, entries.size()); +// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); +// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); +// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); +// } @Test public void testFindByQueryDescOrder() throws Exception { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 01f0ba73fe..ae3d27fdf5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,28 +19,28 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.BDDMockito.willCallRealMethod; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.thingsboard.server.common.data.id.TenantId.SYS_TENANT_ID; +import static org.thingsboard.server.common.data.kv.Aggregation.COUNT; public class AbstractChunkedAggregationTimeseriesDaoTest { - final int START_TS = 1; final int LIMIT = 1; - final int END_TS = 3000; final String TEMP = "temp"; final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; @@ -49,111 +49,104 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Before public void setUp() throws Exception { - tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); + tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - willReturn(Futures.immediateFuture(null)).given(tsDao).getTskvEntriesFuture(any()); - willCallRealMethod().given(tsDao).findAllAsync(any(), any(), any(ReadTsKvQuery.class)); - willCallRealMethod().given(tsDao).findIntervals(any(), any(), any(ReadTsKvQuery.class)); - willCallRealMethod().given(tsDao).getTsForReadTsKvQuery(anyLong(), anyLong()); - willCallRealMethod().given(tsDao).toPartitionTs(anyLong()); - willCallRealMethod().given(tsDao).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any()); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); - - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, 2001, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, 2001, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 2001, 3000 + 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000 + 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2998, 1500, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2999, 3000, 3000, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, END_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 3000, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, 0, 0 + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 0, 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query, START_TS, START_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, query, 1, 2, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() { - long intervalTs = 3; - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - for (long i = START_TS; i <= END_TS; i += intervalTs) { - ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + intervalTs, i + (i + intervalTs - i) / 2, LIMIT, Aggregation.COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, querySub, i, i + intervalTs, Aggregation.COUNT); + for (long i = 1; i <= 3000; i += 3) { + ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, querySub, i, i + 3, COUNT); } } -} \ No newline at end of file +} From 5468d27db8c17a43bf81af664856e9cdb8802415 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 22 Nov 2021 17:38:35 +0200 Subject: [PATCH 08/24] add test for Timeseries Service, where we check findAll() with Aggregation.COUNT. --- .../timeseries/BaseTimeseriesServiceTest.java | 158 +++++++++++++++--- 1 file changed, 132 insertions(+), 26 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 0bfcff6832..672c17614c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -164,32 +164,138 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } -// -// @Test -// public void testFindByQueryOneMilisecondPeriod() throws Exception { -// DeviceId deviceId = new DeviceId(Uuids.timeBased()); -// -// saveEntries(deviceId, TS - 3); -// saveEntries(deviceId, TS - 2); -// saveEntries(deviceId, TS - 1); -// -// List queries = new ArrayList<>(); -// queries.add(new BaseReadTsKvQuery(STRING_KEY, TS, TS, 0, 1000, Aggregation.NONE, "ASC")); -// -// List entries = tsService.findAll(tenantId, deviceId, queries).get(); -// Assert.assertEquals(3, entries.size()); -// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); -// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); -// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); -// -// EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); -// -// entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); -// Assert.assertEquals(3, entries.size()); -// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); -// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); -// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); -// } + + @Test + public void testFindByQueryOneMilisecondPeriod() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS); + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 5 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 5, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 8L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS - 2; i<= TS + 12L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS - 2L; i<= TS + 12L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + } @Test public void testFindByQueryDescOrder() throws Exception { From 38f64a7ac36286c8d20731c7d19b8823ea7c183d Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 26 Nov 2021 16:45:41 +0200 Subject: [PATCH 09/24] refactoring test after code review --- .../timeseries/BaseTimeseriesServiceTest.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 672c17614c..2c5d8b9fea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -166,12 +166,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { } @Test - public void testFindByQueryOneMilisecondPeriod() throws Exception { + public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); saveEntries(deviceId, TS); + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); @@ -187,12 +188,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 10L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); @@ -208,12 +210,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 10L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -231,12 +234,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 8L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -258,8 +262,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { saveEntries(deviceId, i); } - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -281,8 +284,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { saveEntries(deviceId, i); } - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); From cc67e93f972e1e1147fb4f4df1f93b6e77138670 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 29 Nov 2021 11:28:04 +0200 Subject: [PATCH 10/24] change tests cases for more understandable --- .../timeseries/BaseTimeseriesServiceTest.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 2c5d8b9fea..b6ca2d639c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -168,9 +168,9 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); + saveEntries(deviceId, TS - 1L); saveEntries(deviceId, TS); - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 1L); List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); @@ -188,115 +188,115 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 5 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 5, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 8L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 80L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 80L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i=TS - 2; i<= TS + 12L; i++) { + for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i=TS - 2L; i<= TS + 12L; i++) { + for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10L) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test From b54aa55c1ff6c82575b1604257fd1a185c0c7403 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 17 Jan 2022 12:49:36 +0200 Subject: [PATCH 11/24] update tests --- ...ctChunkedAggregationTimeseriesDaoTest.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index ae3d27fdf5..95cd2280fe 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,6 +19,8 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -59,7 +61,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); @@ -69,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -79,8 +81,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2998, 1500, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2999, 3000, 3000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -92,7 +94,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -102,7 +104,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -112,10 +114,11 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, query, 1, 2, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuery, 1, 2, COUNT); } @Test @@ -131,7 +134,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); From d02c7ad7bf0346098bae1786142c053d1f75dc4c Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 17 Jan 2022 18:41:34 +0200 Subject: [PATCH 12/24] fix issue with license --- .../server/dao/timeseries/AggregationTimeseriesDao.java | 2 +- .../dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java index a99ad053b6..b8e42e3165 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 95cd2280fe..68fc8feccc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 51071a7413cf8a3d576af8627806be015869d363 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 19 Jan 2022 18:06:53 +0200 Subject: [PATCH 13/24] remove unnecessary code and add some clean code --- .../server/dao/sqlts/SqlTimeseriesLatestDao.java | 1 - .../server/dao/sqlts/timescale/TimescaleTimeseriesDao.java | 7 ++++++- .../dao/service/timeseries/BaseTimeseriesServiceTest.java | 2 +- .../sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java | 4 ---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 6f58f318a1..7808f0e1bd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -46,7 +46,6 @@ import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index e15ee31b35..15875eb275 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -44,7 +44,12 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Function; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 7677654ec7..04c3425169 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -217,7 +217,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 50 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 68fc8feccc..cab869a647 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,8 +19,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -47,8 +45,6 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; - //SOME PRESENT: When we give data with period l-r, program return data in interval [l;r) - @Before public void setUp() throws Exception { tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); From 40823dd2c0da480e3284a660644ff9fbcfb565f3 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 17:40:37 +0300 Subject: [PATCH 14/24] msa black box tests: redis-cluster test implemented with mvn -DblackBoxTests.redisCluster=true --- docker/cache-redis-cluster.env | 5 + docker/cache-redis.env | 2 + docker/docker-compose.hybrid.yml | 4 - docker/docker-compose.kafka.yml | 4 - docker/docker-compose.postgres.yml | 8 - docker/docker-compose.redis-cluster.yml | 162 ++++++++++++++++++ docker/docker-compose.redis.yml | 102 +++++++++++ docker/docker-compose.yml | 11 +- docker/docker-install-tb.sh | 4 +- docker/docker-remove-services.sh | 2 +- docker/docker-start-services.sh | 2 +- docker/docker-stop-services.sh | 2 +- docker/docker-update-service.sh | 4 +- docker/docker-upgrade-tb.sh | 6 +- docker/tb-coap-transport.env | 3 - docker/tb-http-transport.env | 3 - docker/tb-lwm2m-transport.env | 3 - docker/tb-mqtt-transport.env | 3 - docker/tb-node.env | 2 - docker/tb-snmp-transport.env | 3 - msa/black-box-tests/README.md | 7 +- .../server/msa/ContainerTestSuite.java | 8 +- .../server/msa/ThingsBoardDbInstaller.java | 46 ++++- 23 files changed, 337 insertions(+), 59 deletions(-) create mode 100644 docker/cache-redis-cluster.env create mode 100644 docker/cache-redis.env create mode 100644 docker/docker-compose.redis-cluster.yml create mode 100644 docker/docker-compose.redis.yml diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env new file mode 100644 index 0000000000..de666e003b --- /dev/null +++ b/docker/cache-redis-cluster.env @@ -0,0 +1,5 @@ +CACHE_TYPE=redis +REDIS_CONNECTION_TYPE=cluster +REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 +REDIS_USE_DEFAULT_POOL_CONFIG=false +REDIS_PASSWORD=bitnami \ No newline at end of file diff --git a/docker/cache-redis.env b/docker/cache-redis.env new file mode 100644 index 0000000000..52091ecc31 --- /dev/null +++ b/docker/cache-redis.env @@ -0,0 +1,2 @@ +CACHE_TYPE=redis +REDIS_HOST=redis \ No newline at end of file diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 7d7ab84c8f..e3ba32b779 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -39,7 +39,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-core2: @@ -47,7 +46,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-rule-engine1: @@ -55,7 +53,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-rule-engine2: @@ -63,6 +60,5 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 1683114605..82fbb6ed62 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -36,25 +36,21 @@ services: - queue-kafka.env depends_on: - kafka - - redis tb-core2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine1: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-mqtt-transport1: env_file: - queue-kafka.env diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index 591ea59f7a..8fe8e6f53d 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -31,27 +31,19 @@ services: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-core2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine1: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml new file mode 100644 index 0000000000..e66122f44d --- /dev/null +++ b/docker/docker-compose.redis-cluster.yml @@ -0,0 +1,162 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis cluster + redis-node-0: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-0:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-1: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-1:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-2: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-2:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-3: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-3:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-4: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-4:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-5: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-5:/bitnami/redis/data + depends_on: + - redis-node-0 + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + - 'REDISCLI_AUTH=bitnami' + - 'REDIS_CLUSTER_REPLICAS=1' + - 'REDIS_CLUSTER_CREATOR=yes' + +# ThingsBoard setup to use redis-cluster + tb-core1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-core2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-coap-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-lwm2m-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-snmp-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 +volumes: + redis-cluster_data-0: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_0} + redis-cluster_data-1: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_1} + redis-cluster_data-2: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_2} + redis-cluster_data-3: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_3} + redis-cluster_data-4: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_4} + redis-cluster_data-5: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml new file mode 100644 index 0000000000..671c1952b8 --- /dev/null +++ b/docker/docker-compose.redis.yml @@ -0,0 +1,102 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis standalone + redis: + restart: always + image: bitnami/redis:7.0 + environment: + # ALLOW_EMPTY_PASSWORD is recommended only for development. + ALLOW_EMPTY_PASSWORD: "yes" + ports: + - '6379:6379' + volumes: + - 'redis_data:/bitnami/redis/data' + +# ThingsBoard setup to use redis-standalone + tb-core1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-core2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-coap-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-lwm2m-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-snmp-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor2: + env_file: + - cache-redis.env + depends_on: + - redis + +volumes: + redis_data: + external: true + name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1ba6eda32d..62969b1051 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,15 +26,10 @@ services: environment: ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - redis: - restart: always - image: redis:4.0 - ports: - - "6379" tb-js-executor: restart: always image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" - scale: 20 + scale: 10 env_file: - tb-js-executor.env tb-core1: @@ -59,7 +54,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -85,7 +79,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -109,7 +102,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-rule-engine2: restart: always @@ -131,7 +123,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-mqtt-transport1: restart: always diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 6f6b1e9511..3ab5a873f0 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -48,9 +48,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 89f0f7844c..11bd451aa1 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 9f159774d8..e6b06718c5 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 61e68d6dd5..dd2ccf9ef9 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 739fcf6543..e6ec0c62a4 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -23,5 +23,5 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 3b1415a965..6a4311ffba 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -46,8 +46,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index 9e6a41c930..079443c98b 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 1b4ce7a298..7e0679987f 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -7,6 +7,3 @@ HTTP_REQUEST_TIMEOUT=60000 METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-lwm2m-transport.env b/docker/tb-lwm2m-transport.env index 4616d45972..f284803a46 100644 --- a/docker/tb-lwm2m-transport.env +++ b/docker/tb-lwm2m-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 0cd51c7371..e38cb2124a 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-node.env b/docker/tb-node.env index e3393d41b1..ba66757ecc 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,8 +4,6 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 JS_EVALUATOR=remote TRANSPORT_TYPE=remote -CACHE_TYPE=redis -REDIS_HOST=redis HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false diff --git a/docker/tb-snmp-transport.env b/docker/tb-snmp-transport.env index 4851e9f6c1..e2cc39d658 100644 --- a/docker/tb-snmp-transport.env +++ b/docker/tb-snmp-transport.env @@ -6,6 +6,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index 9c68789e67..6e70de17ca 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -18,8 +18,13 @@ As result, in REPOSITORY column, next images should be present: thingsboard/tb-web-ui thingsboard/tb-js-executor -- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory: +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone: mvn clean install -DblackBoxTests.skip=false +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis cluster: + + mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index f60d6d7545..c9ed8a4040 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -39,7 +39,7 @@ import static org.junit.Assert.fail; @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) @Slf4j public class ContainerTestSuite { - + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; @@ -52,6 +52,7 @@ public class ContainerTestSuite { @ClassRule public static DockerComposeContainer getTestContainer() { if (testContainer == null) { + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); try { final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; @@ -75,7 +76,10 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose.kafka.yml")) + new File(targetDir + "docker-compose.kafka.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.yml") + : new File("./../../docker/docker-compose.redis.yml")) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index de55a3afa3..ac0fa55991 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.msa; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; @@ -24,10 +25,16 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +@Slf4j public class ThingsBoardDbInstaller extends ExternalResource { + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; + private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume"; + private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume"; private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume"; private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume"; @@ -39,6 +46,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final DockerComposeExecutor dockerCompose; private final String postgresDataVolume; + + private final String redisDataVolume; + private final String redisClusterDataVolume; private final String tbLogVolume; private final String tbCoapTransportLogVolume; private final String tbLwm2mTransportLogVolume; @@ -49,14 +59,21 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final Map env; public ThingsBoardDbInstaller() { + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), new File("./../../docker/docker-compose.postgres.yml"), - new File("./../../docker/docker-compose.postgres.volumes.yml")); + new File("./../../docker/docker-compose.postgres.volumes.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.yml") + : new File("./../../docker/docker-compose.redis.yml") + ); String identifier = Base58.randomString(6).toLowerCase(); String project = identifier + Base58.randomString(6).toLowerCase(); postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; + redisDataVolume = project + "_" + REDIS_DATA_VOLUME; + redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME; tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME; tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME; @@ -76,6 +93,13 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); + } + } else { + env.put("REDIS_DATA_VOLUME", redisDataVolume); + } dockerCompose.withEnv(env); } @@ -111,7 +135,20 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); - dockerCompose.withCommand("up -d redis postgres"); + String redisService = ""; + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + redisService = redisService + " redis-node-" + i; + dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i); + dockerCompose.invokeDocker(); + } + } else { + redisService = "redis"; + dockerCompose.withCommand("volume create " + redisDataVolume); + dockerCompose.invokeDocker(); + } + + dockerCompose.withCommand("up -d postgres " + redisService); dockerCompose.invokeCompose(); dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); @@ -137,7 +174,10 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + + (IS_REDIS_CLUSTER + ? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining()) + : redisDataVolume)); dockerCompose.invokeDocker(); } From d7c750f64e60a1dd06aa9436b39cbe9fabbecd13 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 21:30:34 +0300 Subject: [PATCH 15/24] msa black box tests: redis, postgres, node volumes refactored - volumes decoupled to separate files .volumes (used only for tests) --- docker/cache-redis-cluster.env | 2 +- docker/cache-redis.env | 2 +- docker/docker-compose.postgres.volumes.yml | 61 -------------- .../docker-compose.redis-cluster.volumes.yml | 58 +++++++++++++ docker/docker-compose.redis-cluster.yml | 31 ++----- docker/docker-compose.redis.volumes.yml | 27 +++++++ docker/docker-compose.redis.yml | 7 +- docker/docker-compose.volumes.yml | 81 +++++++++++++++++++ .../server/msa/ContainerTestSuite.java | 7 +- .../server/msa/ThingsBoardDbInstaller.java | 9 ++- 10 files changed, 188 insertions(+), 97 deletions(-) create mode 100644 docker/docker-compose.redis-cluster.volumes.yml create mode 100644 docker/docker-compose.redis.volumes.yml create mode 100644 docker/docker-compose.volumes.yml diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env index de666e003b..64058ab556 100644 --- a/docker/cache-redis-cluster.env +++ b/docker/cache-redis-cluster.env @@ -2,4 +2,4 @@ CACHE_TYPE=redis REDIS_CONNECTION_TYPE=cluster REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 REDIS_USE_DEFAULT_POOL_CONFIG=false -REDIS_PASSWORD=bitnami \ No newline at end of file +REDIS_PASSWORD=bitnami diff --git a/docker/cache-redis.env b/docker/cache-redis.env index 52091ecc31..7b92620666 100644 --- a/docker/cache-redis.env +++ b/docker/cache-redis.env @@ -1,2 +1,2 @@ CACHE_TYPE=redis -REDIS_HOST=redis \ No newline at end of file +REDIS_HOST=redis diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 019e087c48..caf78f23d7 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -20,69 +20,8 @@ services: postgres: volumes: - postgres-db-volume:/var/lib/postgresql/data - tb-core1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-core2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-coap-transport: - volumes: - - tb-coap-transport-log-volume:/var/log/tb-coap-transport - tb-lwm2m-transport: - volumes: - - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport - tb-http-transport1: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-http-transport2: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-mqtt-transport1: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-mqtt-transport2: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-snmp-transport: - volumes: - - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport - tb-vc-executor1: - volumes: - - tb-vc-executor-log-volume:/var/log/tb-vc-executor - tb-vc-executor2: - volumes: - - tb-vc-executor-log-volume:/var/log/tb-vc-executor - volumes: postgres-db-volume: external: true name: ${POSTGRES_DATA_VOLUME} - tb-log-volume: - external: true - name: ${TB_LOG_VOLUME} - tb-coap-transport-log-volume: - external: true - name: ${TB_COAP_TRANSPORT_LOG_VOLUME} - tb-lwm2m-transport-log-volume: - external: true - name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} - tb-http-transport-log-volume: - external: true - name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} - tb-mqtt-transport-log-volume: - external: true - name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} - tb-snmp-transport-log-volume: - external: true - name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} - tb-vc-executor-log-volume: - external: true - name: ${TB_VC_EXECUTOR_LOG_VOLUME} \ No newline at end of file diff --git a/docker/docker-compose.redis-cluster.volumes.yml b/docker/docker-compose.redis-cluster.volumes.yml new file mode 100644 index 0000000000..2cf319bd21 --- /dev/null +++ b/docker/docker-compose.redis-cluster.volumes.yml @@ -0,0 +1,58 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + # Redis cluster + redis-node-0: + volumes: + - redis-cluster-data-0:/bitnami/redis/data + redis-node-1: + volumes: + - redis-cluster-data-1:/bitnami/redis/data + redis-node-2: + volumes: + - redis-cluster-data-2:/bitnami/redis/data + redis-node-3: + volumes: + - redis-cluster-data-3:/bitnami/redis/data + redis-node-4: + volumes: + - redis-cluster-data-4:/bitnami/redis/data + redis-node-5: + volumes: + - redis-cluster-data-5:/bitnami/redis/data + +volumes: + redis-cluster-data-0: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_0} + redis-cluster-data-1: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_1} + redis-cluster-data-2: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_2} + redis-cluster-data-3: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_3} + redis-cluster-data-4: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_4} + redis-cluster-data-5: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index e66122f44d..17ca4ed13a 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -21,7 +21,7 @@ services: redis-node-0: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-0:/bitnami/redis/data + - ./tb-node/redis-cluster-data-0:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -29,7 +29,7 @@ services: redis-node-1: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-1:/bitnami/redis/data + - ./tb-node/redis-cluster-data-1:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -37,7 +37,7 @@ services: redis-node-2: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-2:/bitnami/redis/data + - ./tb-node/redis-cluster-data-2:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -45,7 +45,7 @@ services: redis-node-3: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-3:/bitnami/redis/data + - ./tb-node/redis-cluster-data-3:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -53,7 +53,7 @@ services: redis-node-4: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-4:/bitnami/redis/data + - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -61,7 +61,7 @@ services: redis-node-5: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-5:/bitnami/redis/data + - ./tb-node/redis-cluster-data-5:/bitnami/redis/data depends_on: - redis-node-0 - redis-node-1 @@ -141,22 +141,3 @@ services: - cache-redis-cluster.env depends_on: - redis-node-5 -volumes: - redis-cluster_data-0: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_0} - redis-cluster_data-1: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_1} - redis-cluster_data-2: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_2} - redis-cluster_data-3: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_3} - redis-cluster_data-4: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_4} - redis-cluster_data-5: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis.volumes.yml b/docker/docker-compose.redis.volumes.yml new file mode 100644 index 0000000000..090aa441fe --- /dev/null +++ b/docker/docker-compose.redis.volumes.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + redis: + volumes: + - redis-data:/bitnami/redis/data + +volumes: + redis-data: + external: true + name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml index 671c1952b8..e53a974134 100644 --- a/docker/docker-compose.redis.yml +++ b/docker/docker-compose.redis.yml @@ -27,7 +27,7 @@ services: ports: - '6379:6379' volumes: - - 'redis_data:/bitnami/redis/data' + - ./tb-node/redis-data:/bitnami/redis/data # ThingsBoard setup to use redis-standalone tb-core1: @@ -95,8 +95,3 @@ services: - cache-redis.env depends_on: - redis - -volumes: - redis_data: - external: true - name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.volumes.yml b/docker/docker-compose.volumes.yml new file mode 100644 index 0000000000..58269473e4 --- /dev/null +++ b/docker/docker-compose.volumes.yml @@ -0,0 +1,81 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-core1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-core2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-coap-transport: + volumes: + - tb-coap-transport-log-volume:/var/log/tb-coap-transport + tb-lwm2m-transport: + volumes: + - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport + tb-http-transport1: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-http-transport2: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-mqtt-transport1: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-mqtt-transport2: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-snmp-transport: + volumes: + - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + +volumes: + tb-log-volume: + external: true + name: ${TB_LOG_VOLUME} + tb-coap-transport-log-volume: + external: true + name: ${TB_COAP_TRANSPORT_LOG_VOLUME} + tb-lwm2m-transport-log-volume: + external: true + name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} + tb-http-transport-log-volume: + external: true + name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} + tb-mqtt-transport-log-volume: + external: true + name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} + tb-snmp-transport-log-volume: + external: true + name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index c9ed8a4040..cc6661d8d9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -74,12 +74,17 @@ public class ContainerTestSuite { testContainer = new DockerComposeContainerImpl<>( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml")) + : new File("./../../docker/docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") + : new File("./../../docker/docker-compose.redis.volumes.yml") + ) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index ac0fa55991..08a333ab52 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -60,12 +60,17 @@ public class ThingsBoardDbInstaller extends ExternalResource { public ThingsBoardDbInstaller() { log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); - List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), + List composeFiles = Arrays.asList( + new File("./../../docker/docker-compose.yml"), + new File("./../../docker/docker-compose.volumes.yml"), new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.volumes.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml") + : new File("./../../docker/docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") + : new File("./../../docker/docker-compose.redis.volumes.yml") ); String identifier = Base58.randomString(6).toLowerCase(); From 7d7701723cc753a38f5e95a700b66b1fc2414f4e Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 22 Jun 2022 23:09:58 +0300 Subject: [PATCH 16/24] improve logic --- ...stractChunkedAggregationTimeseriesDao.java | 36 ++++--- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 12 --- .../dao/sqlts/AggregationTimeseriesDao.java | 29 ++++++ .../sqlts/BaseAbstractSqlTimeseriesDao.java | 23 +++++ .../dao/sqlts/SqlTimeseriesLatestDao.java | 1 - .../timescale/TimescaleTimeseriesDao.java | 5 - .../timeseries/AggregationTimeseriesDao.java | 97 ------------------- .../CassandraBaseTimeseriesDao.java | 46 +++++---- .../CassandraBaseTimeseriesLatestDao.java | 1 + ...ctChunkedAggregationTimeseriesDaoTest.java | 45 +++++---- 10 files changed, 128 insertions(+), 167 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 57b458a663..d68271798a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -26,6 +26,7 @@ import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -118,12 +119,28 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public long getTsForReadTsKvQuery(long startTs, long endTs) { - return startTs + (endTs - startTs) / 2; + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(entityId, query); + } else { + List>> futures = new ArrayList<>(); + long endPeriod = query.getEndTs(); + long startPeriod = query.getStartTs(); + long step = query.getInterval(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); + long ts = startTs + (endTs - startTs) / 2; + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, subQuery.getKey(), startTs, endTs, ts, query.getAggregation()); + futures.add(aggregateTsKvEntry); + startPeriod = endTs; + } + return getTskvEntriesFuture(Futures.allAsList(futures)); + } } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -136,17 +153,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } - @Override - public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { - return findAndAggregateAsync(entityId, - query.getKey(), - startTs, - endTs, - getTsForReadTsKvQuery(startTs, endTs), - aggregation); - - } - ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index c32123cb94..aa0d9b1c47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; -import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import javax.annotation.Nullable; import java.sql.Connection; @@ -36,7 +35,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.Objects; -import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -122,14 +120,4 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } - - @Override - public Executor getExecutor() { - return service; - } - - @Override - public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { - return interval; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..31270bacc7 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface AggregationTimeseriesDao { + + ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 83a5d11623..dd9d15b8b1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -15,18 +15,25 @@ */ package org.thingsboard.server.dao.sqlts; +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import javax.annotation.Nullable; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; @Slf4j public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { @@ -72,4 +79,20 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni return keyId; } + protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { + return Futures.transform(future, new Function>, List>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, service); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 7808f0e1bd..db30adead2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -45,7 +45,6 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 15875eb275..9c03a9e3dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -165,11 +165,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements super.cleanup(systemTtl); } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - return findAllAsyncWithLimit(entityId, query); - } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java deleted file mode 100644 index b8e42e3165..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.timeseries; - -import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -public interface AggregationTimeseriesDao { - - default ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(tenantId, entityId, query); - } else { - List>> futures = findIntervals(tenantId, entityId, query); - return getTskvEntriesFuture(Futures.allAsList(futures)); - } - } - - default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - List>> futures = new ArrayList<>(); - long endPeriod = query.getEndTs(); - long startPeriod = query.getStartTs(); - long step = query.getInterval(); - while (startPeriod <= endPeriod) { - long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod + 1); - long ts = getTsForReadTsKvQuery(startTs, endTs); - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); - ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); - futures.add(aggregateTsKvEntry); - startPeriod = endTs; - } - return futures; - } - - default long getTsForReadTsKvQuery(long startTs, long endTs) { - return endTs - startTs; - } - - long getIntervalGreaterOrEqualsMinAggregationStep(long interval); - - default ListenableFuture> getTskvEntriesFuture(ListenableFuture>> allAsList) { - return Futures.transform(allAsList, new Function<>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, getExecutor()); - } - - Executor getExecutor(); - - default ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery key, long startTs, long endTs, Aggregation aggregation) { - return Futures.immediateFuture(null); - } - - ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); - - default long toPartitionTs(long ts) { - return ts; - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 8b58e547c9..bb3930b99c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -37,6 +37,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.KvEntry; @@ -45,6 +46,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -59,7 +61,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -265,13 +266,31 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { - return Math.max(interval, MIN_AGGREGATION_STEP_MS); - } - - @Override - public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { - return findAndAggregateAsync(tenantId, entityId, query, startTs, endTs); + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(tenantId, entityId, query); + } else { + long startPeriod = query.getStartTs(); + long endPeriod = query.getEndTs(); + long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); + List>> futures = new ArrayList<>(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); + long ts = endTs - startTs; + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); + startPeriod = endTs; + } + ListenableFuture>> future = Futures.allAsList(futures); + return Futures.transform(future, new Function<>() { + @Nullable + @Override + public List apply(@Nullable List> input) { + return input == null ? Collections.emptyList() : input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList()); + } + }, readResultsProcessingExecutor); + } } @Override @@ -279,8 +298,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD //Cleanup by TTL is native for Cassandra } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); @@ -302,17 +320,11 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return resultFuture; } - @Override - public long toPartitionTs(long ts) { + private long toPartitionTs(long ts) { LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } - @Override - public Executor getExecutor() { - return readResultsProcessingExecutor; - } - private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SimpleListenableFuture> resultFuture) { if (cursor.isFull() || !cursor.hasNextPartition()) { resultFuture.set(cursor.getData()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index e7099db3d2..124af03a1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index cab869a647..db216f7f6e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -59,9 +59,9 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 2001, 3000 + 1, COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT); } @Test @@ -70,8 +70,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000 + 1, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -81,9 +81,9 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000, COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 3000, 3001, COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT); } @@ -93,8 +93,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -103,8 +103,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 0, 1, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 0, 1, getTsForReadTsKvQuery(0, 1), COUNT); } @Test @@ -113,8 +113,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuery, 1, 2, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuery.getKey(), 1, 2, getTsForReadTsKvQuery(1, 2), COUNT); } @Test @@ -123,8 +123,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), Integer.MAX_VALUE, 1L + Integer.MAX_VALUE, getTsForReadTsKvQuery(Integer.MAX_VALUE, 1L + Integer.MAX_VALUE), COUNT); } @Test @@ -133,8 +133,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -142,10 +142,15 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); for (long i = 1; i <= 3000; i += 3) { ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, querySub, i, i + 3, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT); } } + + long getTsForReadTsKvQuery(long startTs, long endTs) { + return startTs + (endTs - startTs) / 2L; + } + } From 7b88a437dc194eaab7dc0f2aeebf7dd6d45e2707 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 23:24:05 +0300 Subject: [PATCH 17/24] docker-compose shell scripts: added CACHE=redis or CACHE=redis-cluster option. Default is redis, README.md updated --- docker/.env | 3 +++ docker/README.md | 7 ++++++ docker/compose-utils.sh | 36 +++++++++++++++++++++++++++-- docker/docker-create-log-folders.sh | 21 +++++++++++++++++ docker/docker-install-tb.sh | 6 +++-- docker/docker-remove-services.sh | 4 +++- docker/docker-start-services.sh | 4 +++- docker/docker-stop-services.sh | 4 +++- docker/docker-update-service.sh | 6 +++-- docker/docker-upgrade-tb.sh | 8 ++++--- 10 files changed, 87 insertions(+), 12 deletions(-) diff --git a/docker/.env b/docker/.env index af0f537603..7bc1a7a6a4 100644 --- a/docker/.env +++ b/docker/.env @@ -1,5 +1,8 @@ TB_QUEUE_TYPE=kafka +# redis or redis-cluster +CACHE=redis + DOCKER_REPO=thingsboard JS_EXECUTOR_DOCKER_NAME=tb-js-executor diff --git a/docker/README.md b/docker/README.md index 01f89ebb6c..71ea87f353 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env` **NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details). +In order to set cache type change the value of `CACHE` variable in `.env` file to one of the following: + +- `redis` - use Redis standalone cache (1 node - 1 master); +- `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves); + +**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details). + Execute the following command to create log folders for the services and chown of these folders to the docker container users. To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access): diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 550f2dfea4..d60c81ec6a 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -73,19 +73,51 @@ function additionalComposeMonitoringArgs() { fi } +function additionalComposeCacheArgs() { + source .env + CACHE_COMPOSE_ARGS="" + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + CACHE_COMPOSE_ARGS="-f docker-compose.redis.yml" + ;; + redis-cluster) + CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 + esac + echo $CACHE_COMPOSE_ARGS +} + function additionalStartupServices() { source .env ADDITIONAL_STARTUP_SERVICES="" case $DATABASE in postgres) - ADDITIONAL_STARTUP_SERVICES=postgres + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres" ;; hybrid) - ADDITIONAL_STARTUP_SERVICES="postgres cassandra" + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra" ;; *) echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 exit 1 esac + + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis" + ;; + redis-cluster) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 + esac + echo $ADDITIONAL_STARTUP_SERVICES } diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index ba945a19df..83257c0e66 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -28,3 +28,24 @@ mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log + +mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/ + +source .env +CACHE="${CACHE:-redis}" +case $CACHE in + redis) + mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/ + ;; + redis-cluster) + mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/ + mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/ + mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/ + mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/ + mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/ + mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/ + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 +esac \ No newline at end of file diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 3ab5a873f0..27e84c4d0b 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -45,12 +45,14 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 11bd451aa1..36e464f45d 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index e6b06718c5..7993ab425b 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index dd2ccf9ef9..ae8a3c0b81 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index e6ec0c62a4..a02f9e208a 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -23,5 +23,7 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 6a4311ffba..c20543783b 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -44,10 +44,12 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 From d903d0ddc0db3a6797552145a5aca4f4bed7e43e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 12:39:50 +0300 Subject: [PATCH 18/24] default Redis cluster password changed, set REDISCLI_AUTH to simplify redis-cli usage with docker exec --- docker/cache-redis-cluster.env | 2 +- docker/docker-compose.redis-cluster.yml | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env index 64058ab556..a3b516063b 100644 --- a/docker/cache-redis-cluster.env +++ b/docker/cache-redis-cluster.env @@ -2,4 +2,4 @@ CACHE_TYPE=redis REDIS_CONNECTION_TYPE=cluster REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 REDIS_USE_DEFAULT_POOL_CONFIG=false -REDIS_PASSWORD=bitnami +REDIS_PASSWORD=thingsboard diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index 17ca4ed13a..0a11b765e8 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -23,7 +23,8 @@ services: volumes: - ./tb-node/redis-cluster-data-0:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-1: @@ -31,7 +32,8 @@ services: volumes: - ./tb-node/redis-cluster-data-1:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-2: @@ -39,7 +41,8 @@ services: volumes: - ./tb-node/redis-cluster-data-2:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-3: @@ -47,7 +50,8 @@ services: volumes: - ./tb-node/redis-cluster-data-3:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-4: @@ -55,7 +59,7 @@ services: volumes: - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-5: @@ -69,9 +73,9 @@ services: - redis-node-3 - redis-node-4 environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' - - 'REDISCLI_AUTH=bitnami' - 'REDIS_CLUSTER_REPLICAS=1' - 'REDIS_CLUSTER_CREATOR=yes' From 59b04e6b56b9c957f44d7683b02eb08c1279983e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 12:42:17 +0300 Subject: [PATCH 19/24] REDISCLI_AUTH password --- docker/docker-compose.redis-cluster.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index 0a11b765e8..55c1c3f1db 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -60,6 +60,7 @@ services: - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-5: From 4f8806dd8bc98995f7c1c97d0a415486d20fa5ec Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 23 Jun 2022 13:47:28 +0300 Subject: [PATCH 20/24] fix tests in nosql --- .../timeseries/BaseTimeseriesServiceTest.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 04c3425169..b38afe77f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -230,92 +230,92 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); - for (long i = TS; i <= TS + 100L; i += 10L) { + for (long i = TS; i <= TS + 100000L; i += 10000L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS + 100L + 1L); + saveEntries(deviceId, TS + 100000L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); - for (long i = TS; i <= TS + 80L; i += 10L) { + for (long i = TS; i <= TS + 80000L; i += 10000L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS + 80L + 1L); + saveEntries(deviceId, TS + 80000L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10) { + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10L) { + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test From 472357109dce21ed3aabcc30b747acfb962339e0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 23 Jun 2022 14:12:03 +0300 Subject: [PATCH 21/24] VC: Detect default remote branch. Fix Entities Version Control REST API mappings to handle branch parameter. --- .../EntitiesVersionControlController.java | 68 +++++++++---------- .../DefaultEntitiesVersionControlService.java | 5 +- .../DefaultGitVersionControlQueueService.java | 9 ++- .../vc/EntitiesVersionControlService.java | 11 ++- .../vc/GitVersionControlQueueService.java | 7 +- .../sync/vc/data/ListBranchesGitRequest.java | 3 +- common/cluster-api/src/main/proto/queue.proto | 7 +- .../common/data/sync/vc/BranchInfo.java | 39 +++++++++++ .../DefaultClusterVersionControlService.java | 6 +- .../sync/vc/DefaultGitRepositoryService.java | 7 +- .../server/service/sync/vc/GitRepository.java | 31 +++++++-- .../service/sync/vc/GitRepositoryService.java | 5 +- .../http/entities-version-control.service.ts | 12 ++-- 13 files changed, 142 insertions(+), 68 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 1bcc304548..fff3c6a4bb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import lombok.Data; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; @@ -39,6 +38,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -56,6 +56,7 @@ import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; @@ -138,10 +139,10 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profile 1 version 1.0\"\n" + " }\n" + "]\n```") - @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) - public DeferredResult> listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, + @GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"}) + public DeferredResult> listEntityVersions(@PathVariable EntityType entityType, @PathVariable UUID externalEntityUuid, + @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -165,9 +166,9 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profiles from dev\"\n" + " }\n" + "]\n```") - @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) - public DeferredResult> listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType, + @GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"}) + public DeferredResult> listEntityTypeVersions(@PathVariable EntityType entityType, + @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -198,8 +199,8 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Devices added\"\n" + " }\n" + "]\n```") - @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) - public DeferredResult> listVersions(@PathVariable String branch, + @GetMapping(value = "/version", params = {"branch", "pageSize", "page"}) + public DeferredResult> listVersions(@RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -216,17 +217,17 @@ public class EntitiesVersionControlController extends BaseController { } - @GetMapping("/entity/{branch}/{entityType}/{versionId}") - public DeferredResult> listEntitiesAtVersion(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable String versionId) throws Exception { + @GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"}) + public DeferredResult> listEntitiesAtVersion(@PathVariable EntityType entityType, + @PathVariable String versionId, + @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); } - @GetMapping("/entity/{branch}/{versionId}") - public DeferredResult> listAllEntitiesAtVersion(@PathVariable String branch, - @PathVariable String versionId) throws Exception { + @GetMapping(value = "/entity/{versionId}", params = {"branch"}) + public DeferredResult> listAllEntitiesAtVersion(@PathVariable String versionId, + @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); } @@ -240,10 +241,10 @@ public class EntitiesVersionControlController extends BaseController { return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); } - @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") - public DeferredResult compareEntityDataToVersion(@PathVariable String branch, - @PathVariable EntityType entityType, + @GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"}) + public DeferredResult compareEntityDataToVersion(@PathVariable EntityType entityType, @PathVariable UUID internalEntityUuid, + @RequestParam String branch, @RequestParam String versionId) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); @@ -318,20 +319,21 @@ public class EntitiesVersionControlController extends BaseController { try { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); final TenantId tenantId = getTenantId(); - ListenableFuture> branches = versionControlService.listBranches(tenantId); + ListenableFuture> branches = versionControlService.listBranches(tenantId); return wrapFuture(Futures.transform(branches, remoteBranches -> { List infos = new ArrayList<>(); - - String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); - if (StringUtils.isNotEmpty(defaultBranch)) { - infos.add(new BranchInfo(defaultBranch, true)); + BranchInfo defaultBranch; + String defaultBranchName = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranchName)) { + defaultBranch = new BranchInfo(defaultBranchName, true); + } else { + defaultBranch = remoteBranches.stream().filter(BranchInfo::isDefault).findFirst().orElse(null); } - - remoteBranches.forEach(branch -> { - if (!branch.equals(defaultBranch)) { - infos.add(new BranchInfo(branch, false)); - } - }); + if (defaultBranch != null) { + infos.add(defaultBranch); + } + infos.addAll(remoteBranches.stream().filter(b -> !b.equals(defaultBranch)) + .map(b -> new BranchInfo(b.getName(), false)).collect(Collectors.toList())); return infos; }, MoreExecutors.directExecutor())); } catch (Exception e) { @@ -339,10 +341,4 @@ public class EntitiesVersionControlController extends BaseController { } } - @Data - public static class BranchInfo { - private final String name; - private final boolean isDefault; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 6ea8115e53..3b625fc086 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,12 +25,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; -import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.cache.CaffeineTbTransactionalCache; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -49,6 +47,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityLoadError; @@ -479,7 +478,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + public ListenableFuture> listBranches(TenantId tenantId) throws Exception { return gitServiceQueue.listBranches(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 99246d282f..628605824e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; @@ -240,7 +241,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture> listBranches(TenantId tenantId) { + public ListenableFuture> listBranches(TenantId tenantId) { ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); } @@ -382,7 +383,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu ((CommitGitRequest) request).getFuture().set(commitResult); } else if (vcResponseMsg.hasListBranchesResponse()) { var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); - ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList()); + ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList().stream().map(this::getBranchInfo).collect(Collectors.toList())); } else if (vcResponseMsg.hasListEntitiesResponse()) { var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); ((ListEntitiesGitRequest) request).getFuture().set( @@ -439,6 +440,10 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); } + private BranchInfo getBranchInfo(TransportProtos.BranchInfoProto proto) { + return new BranchInfo(proto.getName(), proto.getIsDefault()); + } + @SuppressWarnings("rawtypes") @SneakyThrows private EntityExportData toData(String data) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 3ae68e33d0..5aa4c40008 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,22 +16,21 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; -import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; -import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -60,7 +59,7 @@ public interface EntitiesVersionControlService { ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; - ListenableFuture> listBranches(TenantId tenantId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; RepositorySettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index a1aad04701..a1e64da8f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -24,14 +24,15 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; -import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import java.util.List; @@ -55,7 +56,7 @@ public interface GitVersionControlQueueService { ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); - ListenableFuture> listBranches(TenantId tenantId); + ListenableFuture> listBranches(TenantId tenantId); ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java index c045030dc7..4d89efa14d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java @@ -16,10 +16,11 @@ package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import java.util.List; -public class ListBranchesGitRequest extends PendingGitRequest> { +public class ListBranchesGitRequest extends PendingGitRequest> { public ListBranchesGitRequest(TenantId tenantId) { super(tenantId); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 3079978bce..b7362c7f03 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -765,8 +765,13 @@ message ListEntitiesResponseMsg { message ListBranchesRequestMsg { } +message BranchInfoProto { + string name = 1; + bool isDefault = 2; +} + message ListBranchesResponseMsg { - repeated string branches = 1; + repeated BranchInfoProto branches = 1; } message EntityContentRequestMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java new file mode 100644 index 0000000000..fcf1f4470e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.Data; + +import java.util.Objects; + +@Data +public class BranchInfo { + private final String name; + private final boolean isDefault; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BranchInfo that = (BranchInfo) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index b28618372e..f74f18ddaa 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AddMsg; +import org.thingsboard.server.gen.transport.TransportProtos.BranchInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg; @@ -332,7 +333,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { - var branches = vcService.listBranches(ctx.getTenantId()); + var branches = vcService.listBranches(ctx.getTenantId()).stream() + .map(branchInfo -> BranchInfoProto.newBuilder() + .setName(branchInfo.getName()) + .setIsDefault(branchInfo.isDefault()).build()).collect(Collectors.toList()); reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 99dbebc3c9..d3615f7228 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -29,8 +29,9 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.GitRepository.Diff; @@ -84,7 +85,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); repository.resetAndClean(); - if (repository.listRemoteBranches().contains(branch)) { + if (repository.listRemoteBranches().contains(new BranchInfo(branch, false))) { repository.merge(branch); } } catch (IOException | GitAPIException gitAPIException) { @@ -182,7 +183,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listBranches(TenantId tenantId) { + public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); try { return repository.listRemoteBranches(); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 2b2802d003..392c686a14 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -41,10 +41,12 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; @@ -58,6 +60,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; @@ -69,7 +72,12 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.PublicKey; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -84,6 +92,8 @@ public class GitRepository { @Getter private final String directory; + private ObjectId headId; + private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.settings = settings; @@ -135,8 +145,12 @@ public class GitRepository { } public void fetch() throws GitAPIException { - execute(git.fetch() + FetchResult result = execute(git.fetch() .setRemoveDeletedRefs(true)); + Ref head = result.getAdvertisedRef(Constants.HEAD); + if (head != null) { + this.headId = head.getObjectId(); + } } public void deleteLocalBranchIfExists(String branch) throws GitAPIException { @@ -162,13 +176,11 @@ public class GitRepository { .include(branchId)); } - - public List listRemoteBranches() throws GitAPIException { + public List listRemoteBranches() throws GitAPIException { return execute(git.branchList() .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .filter(ref -> !ref.getName().equals(Constants.HEAD)) - .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) - .map(name -> StringUtils.removeStart(name, "origin/")) + .map(this::toBranchInfo) .distinct().collect(Collectors.toList()); } @@ -325,6 +337,13 @@ public class GitRepository { .collect(Collectors.toList()); } + private BranchInfo toBranchInfo(Ref ref) { + String name = org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()); + String branchName = StringUtils.removeStart(name, "origin/"); + boolean isDefault = this.headId != null && this.headId.equals(ref.getObjectId()); + return new BranchInfo(branchName, isDefault); + } + private Commit toCommit(RevCommit revCommit) { return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 057fa9c1ab..d4f909df9e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -19,8 +19,9 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.GitRepository.Diff; @@ -57,7 +58,7 @@ public interface GitRepositoryService { void abort(PendingCommit commit); - List listBranches(TenantId tenantId); + List listBranches(TenantId tenantId); String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index ec0b6ba2ae..d7e892f1db 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -118,20 +118,23 @@ export class EntitiesVersionControlService { public listEntityVersions(pageLink: PageLink, branch: string, externalEntityId: EntityId, config?: RequestConfig): Observable> { - return this.http.get>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}&branch=${encodedBranch}`, defaultHttpOptionsFromConfig(config)); } public listEntityTypeVersions(pageLink: PageLink, branch: string, entityType: EntityType, config?: RequestConfig): Observable> { - return this.http.get>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version/${entityType}${pageLink.toQuery()}&branch=${encodedBranch}`, defaultHttpOptionsFromConfig(config)); } public listVersions(pageLink: PageLink, branch: string, config?: RequestConfig): Observable> { - return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version${pageLink.toQuery()}&branch=${encodedBranch}`, defaultHttpOptionsFromConfig(config)); } @@ -160,7 +163,8 @@ export class EntitiesVersionControlService { entityId: EntityId, versionId: string, config?: RequestConfig): Observable { - return this.http.get(`/api/entities/vc/diff/${branch}/${entityId.entityType}/${entityId.id}?versionId=${versionId}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get(`/api/entities/vc/diff/${entityId.entityType}/${entityId.id}?branch=${encodedBranch}&versionId=${versionId}`, defaultHttpOptionsFromConfig(config)); } From 99de983635a4b5f49d3de77832d31352deabad02 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 15:04:03 +0300 Subject: [PATCH 22/24] docker-compose: bump zookeeper version from 3.5 up to 3.8.0, disable new admin feature (ZOO_ADMINSERVER_ENABLED=false) --- docker/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 62969b1051..3e0f21391d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,12 +20,13 @@ version: '2.2' services: zookeeper: restart: always - image: "zookeeper:3.5" + image: "zookeeper:3.8.0" ports: - "2181" environment: ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 + ZOO_ADMINSERVER_ENABLED: "false" tb-js-executor: restart: always image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" From 46682b8bb6cb166e30e6226ae54bbcbc003c3368 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 15:05:41 +0300 Subject: [PATCH 23/24] docker-compose: cleanup depends_on for external services --- docker/docker-compose.aws-sqs.yml | 26 -------------------------- docker/docker-compose.confluent.yml | 8 -------- 2 files changed, 34 deletions(-) diff --git a/docker/docker-compose.aws-sqs.yml b/docker/docker-compose.aws-sqs.yml index e50d0d424f..dff3f595ad 100644 --- a/docker/docker-compose.aws-sqs.yml +++ b/docker/docker-compose.aws-sqs.yml @@ -23,59 +23,33 @@ services: tb-core1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index 3d5abd0abe..077acdca98 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -23,23 +23,15 @@ services: tb-core1: env_file: - queue-confluent.env - depends_on: - - redis tb-core2: env_file: - queue-confluent.env - depends_on: - - redis tb-rule-engine1: env_file: - queue-confluent.env - depends_on: - - redis tb-rule-engine2: env_file: - queue-confluent.env - depends_on: - - redis tb-mqtt-transport1: env_file: - queue-confluent.env From dd2d1e6443751c2ef8c407ee092998bd438135f0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jun 2022 15:38:31 +0300 Subject: [PATCH 24/24] Removed redundant variable --- .../dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index d68271798a..69f16d4bbc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -131,8 +131,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq long startTs = startPeriod; long endTs = Math.min(startPeriod + step, endPeriod + 1); long ts = startTs + (endTs - startTs) / 2; - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); - ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, subQuery.getKey(), startTs, endTs, ts, query.getAggregation()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); futures.add(aggregateTsKvEntry); startPeriod = endTs; }