From 32c18a882ab7a03531ca1851ac68a0b94531b41f Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 17 Nov 2021 15:43:35 +0200 Subject: [PATCH 01/80] Aggregation tsDao test: givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs --- ...stractChunkedAggregationTimeseriesDao.java | 2 +- ...ctChunkedAggregationTimeseriesDaoTest.java | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 9c2dce109b..4d023c6215 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -146,7 +146,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } - private ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); return Futures.transform(setFutures(entitiesFutures), entity -> { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java new file mode 100644 index 0000000000..b8e59daa17 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -0,0 +1,44 @@ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class AbstractChunkedAggregationTimeseriesDaoTest { + + AbstractChunkedAggregationTimeseriesDao tsDao; + + @Before + public void setUp() throws Exception { + tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); + ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); + willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { + ReadTsKvQuery query = new BaseReadTsKvQuery("temp", 0, 3000, 2000, 0, Aggregation.COUNT, "DESC"); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 0, 2000, 1000, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 2001, 3000, 2500, Aggregation.COUNT); + } +} \ No newline at end of file From aa88ec774e7f1d990641083589b3391dfb59620a Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 17 Nov 2021 16:43:40 +0200 Subject: [PATCH 02/80] add test with must use case --- ...ctChunkedAggregationTimeseriesDaoTest.java | 87 ++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index b8e59daa17..57706e61bd 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -23,6 +23,11 @@ import static org.mockito.Mockito.verify; public class AbstractChunkedAggregationTimeseriesDaoTest { + final int START_TS = 1; + final int LIMIT = 0; + final int END_TS = 3000; + final String TEMP = "temp"; + final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; @Before @@ -34,11 +39,87 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { - ReadTsKvQuery query = new BaseReadTsKvQuery("temp", 0, 3000, 2000, 0, Aggregation.COUNT, "DESC"); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 0, 2000, 1000, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, "temp", 2001, 3000, 2500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 2000, 1000, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 2999, 1499, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsLongMax() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Long.MAX_VALUE, Long.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Long.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { + long intervalTs = 3; + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + for (long i = START_TS; i < END_TS; i += intervalTs) { + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, i, i + intervalTs, i, Aggregation.COUNT); + } } } \ No newline at end of file From d4d61d3e9b50ceafe709d55bb56da199124bb391 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 18 Nov 2021 16:19:06 +0200 Subject: [PATCH 03/80] refactoring code --- ...stractChunkedAggregationTimeseriesDao.java | 32 ++++++------- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 14 +++++- .../sqlts/BaseAbstractSqlTimeseriesDao.java | 24 ---------- .../dao/sqlts/SqlTimeseriesLatestDao.java | 1 + .../timescale/TimescaleTimeseriesDao.java | 10 ++-- .../CassandraBaseTimeseriesDao.java | 46 ++++++++----------- .../CassandraBaseTimeseriesLatestDao.java | 1 - ...ctChunkedAggregationTimeseriesDaoTest.java | 41 +++++++++++------ 8 files changed, 80 insertions(+), 89 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 4d023c6215..c7e9796eef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; @@ -115,24 +114,12 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(entityId, query); - } else { - long stepTs = query.getStartTs(); - List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + query.getInterval(); - long ts = startTs + (endTs - startTs) / 2; - futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); - stepTs = endTs; - } - return getTskvEntriesFuture(Futures.allAsList(futures)); - } + public long getTsForReadTsKvQuery(long startTs, long endTs) { + return startTs + (endTs - startTs) / 2; } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -146,6 +133,17 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } + @Override + public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { + return findAndAggregateAsync(entityId, + query.getKey(), + startTs, + endTs, + getTsForReadTsKvQuery(startTs, endTs), + aggregation); + + } + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index a5eee9c0f4..de0d3df4ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; +import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import javax.annotation.Nullable; import java.sql.Connection; @@ -35,6 +36,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.Objects; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -91,7 +93,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries .stream() .map(query -> findAllAsync(tenantId, entityId, query)) .collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { @@ -120,4 +122,14 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } + + @Override + public Executor getExecutor() { + return service; + } + + @Override + public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { + return interval; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 7e7f2ca00c..a6c6e5e61c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -15,33 +15,24 @@ */ package org.thingsboard.server.dao.sqlts; -import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; -import javax.annotation.Nullable; -import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; @Slf4j public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); - protected static final ReentrantLock tsCreationLock = new ReentrantLock(); - @Autowired protected TsKvDictionaryRepository dictionaryRepository; @@ -81,19 +72,4 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni return keyId; } - protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, service); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 3937b7c2c5..52d3ddf53b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -45,6 +45,7 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; +import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 435c9c5b70..15c211e6d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -34,7 +33,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; @@ -46,9 +44,6 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.sql.CallableStatement; -import java.sql.SQLException; -import java.sql.Types; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -165,6 +160,11 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements super.cleanup(systemTtl); } + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + return findAllAsyncWithLimit(entityId, query); + } + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 736b234eb9..3ebe14fdc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -37,7 +37,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.KvEntry; @@ -46,7 +45,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; -import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -61,6 +59,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -141,7 +140,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { @@ -266,29 +265,13 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(tenantId, entityId, query); - } else { - long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); - long stepTs = query.getStartTs(); - List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + step; - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder()); - futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); - stepTs = endTs; - } - ListenableFuture>> future = Futures.allAsList(futures); - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> input) { - return input == null ? Collections.emptyList() : input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList()); - } - }, readResultsProcessingExecutor); - } + public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { + return Math.max(interval, MIN_AGGREGATION_STEP_MS); + } + + @Override + public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { + return findAndAggregateAsync(tenantId, entityId, query, startTs, endTs); } @Override @@ -296,7 +279,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD //Cleanup by TTL is native for Cassandra } - private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); @@ -318,11 +302,17 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return resultFuture; } - private long toPartitionTs(long ts) { + @Override + public long toPartitionTs(long ts) { LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } + @Override + public Executor getExecutor() { + return readResultsProcessingExecutor; + } + private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SimpleListenableFuture> resultFuture) { if (cursor.isFull() || !cursor.hasNextPartition()) { resultFuture.set(cursor.getData()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 0b4ae71e85..c3b9d0cbb6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; -import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import javax.annotation.Nullable; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 57706e61bd..5fed9083e6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.sqlts; import com.google.common.util.concurrent.Futures; @@ -37,7 +52,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -47,7 +62,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -56,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -66,7 +81,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -75,7 +90,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -84,7 +99,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -93,25 +108,25 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); } - @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsLongMax() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Long.MAX_VALUE, Long.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + //@Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Long.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); } - @Test + //@Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); From 729663e26d115ada79c9d44a9ccb80d41fd316a9 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 18 Nov 2021 16:21:55 +0200 Subject: [PATCH 04/80] refactoring code --- .../dao/sqlts/AggregationTimeseriesDao.java | 29 ------ .../timeseries/AggregationTimeseriesDao.java | 96 +++++++++++++++++++ ...ctChunkedAggregationTimeseriesDaoTest.java | 18 ++-- 3 files changed, 105 insertions(+), 38 deletions(-) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java deleted file mode 100644 index 4464428200..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts; - -import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import java.util.List; - -public interface AggregationTimeseriesDao { - - ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..bf6948b1ac --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -0,0 +1,96 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.stream.Collectors; + +public interface AggregationTimeseriesDao { + + default ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(tenantId, entityId, query); + } else { + long step = getIntervalGreaterOrEqualsMinAggregationStep(query.getInterval()); + long stepTs = query.getStartTs(); + List>> futures = findIntervals(tenantId, entityId, query, step, stepTs); + return getTskvEntriesFuture(Futures.allAsList(futures)); + } + } + + default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long step, long stepTs) { + List>> futures = new ArrayList<>(); + while (stepTs < query.getEndTs()) { + long startTs = stepTs; + long endTs = stepTs + step; + long ts = getTsForReadTsKvQuery(startTs, endTs); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); + futures.add(aggregateTsKvEntry); + stepTs = endTs; + } + return futures; + } + + default long getTsForReadTsKvQuery(long startTs, long endTs) { + return endTs - startTs; + } + + long getIntervalGreaterOrEqualsMinAggregationStep(long interval); + + default ListenableFuture> getTskvEntriesFuture(ListenableFuture>> allAsList) { + return Futures.transform(allAsList, new Function<>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, getExecutor()); + } + + Executor getExecutor(); + + default ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery key, long startTs, long endTs, Aggregation aggregation) { + return Futures.immediateFuture(null); + } + + ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); + + default long toPartitionTs(long ts) { + return ts; + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 5fed9083e6..43c2467885 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -52,7 +52,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -62,7 +62,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -71,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -81,7 +81,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -90,7 +90,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -99,7 +99,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -108,7 +108,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -117,7 +117,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -126,7 +126,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); } - //@Test + @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); From 76c7b34c3b242c8c98427e8a093c4c8df3d7e13c Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 19 Nov 2021 14:03:30 +0200 Subject: [PATCH 05/80] * refactoring test and code * fix bug with interval in findAllAsync() --- .../timeseries/AggregationTimeseriesDao.java | 17 ++--- ...ctChunkedAggregationTimeseriesDaoTest.java | 69 ++++++++++++------- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java index bf6948b1ac..a99ad053b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -38,23 +38,24 @@ public interface AggregationTimeseriesDao { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { - long step = getIntervalGreaterOrEqualsMinAggregationStep(query.getInterval()); - long stepTs = query.getStartTs(); - List>> futures = findIntervals(tenantId, entityId, query, step, stepTs); + List>> futures = findIntervals(tenantId, entityId, query); return getTskvEntriesFuture(Futures.allAsList(futures)); } } - default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long step, long stepTs) { + default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + step; + long endPeriod = query.getEndTs(); + long startPeriod = query.getStartTs(); + long step = query.getInterval(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); long ts = getTsForReadTsKvQuery(startTs, endTs); ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); futures.add(aggregateTsKvEntry); - stepTs = endTs; + startPeriod = endTs; } return futures; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 43c2467885..b8a2328fe2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -39,91 +39,109 @@ import static org.mockito.Mockito.verify; public class AbstractChunkedAggregationTimeseriesDaoTest { final int START_TS = 1; - final int LIMIT = 0; + final int LIMIT = 1; final int END_TS = 3000; final String TEMP = "temp"; final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; + //SOME PRESENT: When we give data with period l-r, program return data in interval [l;r) + @Before public void setUp() throws Exception { tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(null)).given(tsDao).getTskvEntriesFuture(any()); + willCallRealMethod().given(tsDao).findAllAsync(any(), any(), any(ReadTsKvQuery.class)); + willCallRealMethod().given(tsDao).findIntervals(any(), any(), any(ReadTsKvQuery.class)); + willCallRealMethod().given(tsDao).getTsForReadTsKvQuery(anyLong(), anyLong()); + willCallRealMethod().given(tsDao).toPartitionTs(anyLong()); + willCallRealMethod().given(tsDao).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); + tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 2000, 1000, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 2001, END_TS, 2500, Aggregation.COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, 2001, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, 2001, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, END_TS, 1500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 2999, 1499, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, END_TS, END_TS, END_TS, Aggregation.COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, END_TS, END_TS + 1, Aggregation.COUNT); + } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1501, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 0, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 0, 0, 0, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, 0, 0 + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, 1, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, 1, 1, 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query, START_TS, START_TS + 1, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Aggregation.COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, START_TS, END_TS, 1500, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); } @Test @@ -132,9 +150,10 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - verify(tsDao, times(1000)).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - for (long i = START_TS; i < END_TS; i += intervalTs) { - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TEMP, i, i + intervalTs, i, Aggregation.COUNT); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + for (long i = START_TS; i <= END_TS; i += intervalTs) { + ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + intervalTs, i + (i + intervalTs - i) / 2, LIMIT, Aggregation.COUNT, DESC); + verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, querySub, i, i + intervalTs, Aggregation.COUNT); } } } \ No newline at end of file From 32fb91bce560015d8d02fd6a8fafe5b34a1300b1 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 19 Nov 2021 15:09:52 +0200 Subject: [PATCH 06/80] fixed an issue with grammar mistake in tests name --- ...actChunkedAggregationTimeseriesDaoTest.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index b8a2328fe2..01f0ba73fe 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -61,7 +61,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanLastIntervalShorterThanOthersAndEqualsEndTs() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); @@ -73,7 +73,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriod() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -83,7 +83,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodMinusOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); @@ -96,7 +96,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsPeriodPlusOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -106,7 +106,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsZero() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -116,7 +116,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsOne() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -125,7 +125,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -135,7 +135,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanIntervalEqualsBigNumber() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); @@ -145,7 +145,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { } @Test - public void givenIntervalNotMultiplePeriod_whenAggregateCount_thanCountIntervalEqualsPeriodSize() { + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() { long intervalTs = 3; ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); From df2758e8e4bdac03687a95a2f4d420e2198ce7ae Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 22 Nov 2021 12:53:30 +0200 Subject: [PATCH 07/80] refactoring test and start add test in service level --- .../timeseries/BaseTimeseriesServiceTest.java | 26 ++++ ...ctChunkedAggregationTimeseriesDaoTest.java | 117 ++++++++---------- 2 files changed, 81 insertions(+), 62 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index b7872fc239..0bfcff6832 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -164,6 +164,32 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } +// +// @Test +// public void testFindByQueryOneMilisecondPeriod() throws Exception { +// DeviceId deviceId = new DeviceId(Uuids.timeBased()); +// +// saveEntries(deviceId, TS - 3); +// saveEntries(deviceId, TS - 2); +// saveEntries(deviceId, TS - 1); +// +// List queries = new ArrayList<>(); +// queries.add(new BaseReadTsKvQuery(STRING_KEY, TS, TS, 0, 1000, Aggregation.NONE, "ASC")); +// +// List entries = tsService.findAll(tenantId, deviceId, queries).get(); +// Assert.assertEquals(3, entries.size()); +// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); +// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); +// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); +// +// EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); +// +// entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); +// Assert.assertEquals(3, entries.size()); +// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); +// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); +// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); +// } @Test public void testFindByQueryDescOrder() throws Exception { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 01f0ba73fe..ae3d27fdf5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,28 +19,28 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.BDDMockito.willCallRealMethod; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.thingsboard.server.common.data.id.TenantId.SYS_TENANT_ID; +import static org.thingsboard.server.common.data.kv.Aggregation.COUNT; public class AbstractChunkedAggregationTimeseriesDaoTest { - final int START_TS = 1; final int LIMIT = 1; - final int END_TS = 3000; final String TEMP = "temp"; final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; @@ -49,111 +49,104 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Before public void setUp() throws Exception { - tsDao = mock(AbstractChunkedAggregationTimeseriesDao.class); + tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); - willReturn(Futures.immediateFuture(null)).given(tsDao).getTskvEntriesFuture(any()); - willCallRealMethod().given(tsDao).findAllAsync(any(), any(), any(ReadTsKvQuery.class)); - willCallRealMethod().given(tsDao).findIntervals(any(), any(), any(ReadTsKvQuery.class)); - willCallRealMethod().given(tsDao).getTsForReadTsKvQuery(anyLong(), anyLong()); - willCallRealMethod().given(tsDao).toPartitionTs(anyLong()); - willCallRealMethod().given(tsDao).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any()); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2000, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, 2001, START_TS + (2001 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, END_TS, 2001 + (END_TS + 1 - 2001) / 2, LIMIT, Aggregation.COUNT, DESC); - - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, 2001, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, 2001, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 2001, 3000 + 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, END_TS, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000 + 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 2999, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS - 2, START_TS + (END_TS - 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, END_TS - 1, END_TS, END_TS - 1 + (END_TS + 1 - (END_TS - 1)) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2998, 1500, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2999, 3000, 3000, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS, Aggregation.COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQuerySecond, END_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 3000, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, 3001, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (3001 + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, 0, 0 + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 0, 1, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, START_TS, 1, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query, START_TS, START_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, query, 1, 2, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE + (Integer.MAX_VALUE + 1L - Integer.MAX_VALUE) / 2L, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, Integer.MAX_VALUE, LIMIT, Aggregation.COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, START_TS + (END_TS + 1 - START_TS) / 2, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, subQueryFirst, START_TS, END_TS + 1, Aggregation.COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); } @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() { - long intervalTs = 3; - ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, START_TS, END_TS, intervalTs, LIMIT, Aggregation.COUNT, DESC); - willCallRealMethod().given(tsDao).findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); - tsDao.findAllAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, query); + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - for (long i = START_TS; i <= END_TS; i += intervalTs) { - ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + intervalTs, i + (i + intervalTs - i) / 2, LIMIT, Aggregation.COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, querySub, i, i + intervalTs, Aggregation.COUNT); + for (long i = 1; i <= 3000; i += 3) { + ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, querySub, i, i + 3, COUNT); } } -} \ No newline at end of file +} From 5468d27db8c17a43bf81af664856e9cdb8802415 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 22 Nov 2021 17:38:35 +0200 Subject: [PATCH 08/80] add test for Timeseries Service, where we check findAll() with Aggregation.COUNT. --- .../timeseries/BaseTimeseriesServiceTest.java | 158 +++++++++++++++--- 1 file changed, 132 insertions(+), 26 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 0bfcff6832..672c17614c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -164,32 +164,138 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } -// -// @Test -// public void testFindByQueryOneMilisecondPeriod() throws Exception { -// DeviceId deviceId = new DeviceId(Uuids.timeBased()); -// -// saveEntries(deviceId, TS - 3); -// saveEntries(deviceId, TS - 2); -// saveEntries(deviceId, TS - 1); -// -// List queries = new ArrayList<>(); -// queries.add(new BaseReadTsKvQuery(STRING_KEY, TS, TS, 0, 1000, Aggregation.NONE, "ASC")); -// -// List entries = tsService.findAll(tenantId, deviceId, queries).get(); -// Assert.assertEquals(3, entries.size()); -// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); -// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); -// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); -// -// EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); -// -// entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); -// Assert.assertEquals(3, entries.size()); -// Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); -// Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); -// Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); -// } + + @Test + public void testFindByQueryOneMilisecondPeriod() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS); + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 5 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 5, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS; i<= TS + 8L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS - 2; i<= TS + 12L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i=TS - 2L; i<= TS + 12L; i++) { + saveEntries(deviceId, i); + } + + List queries = new ArrayList<>(); + queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + } @Test public void testFindByQueryDescOrder() throws Exception { From 38f64a7ac36286c8d20731c7d19b8823ea7c183d Mon Sep 17 00:00:00 2001 From: van-vanich Date: Fri, 26 Nov 2021 16:45:41 +0200 Subject: [PATCH 09/80] refactoring test after code review --- .../timeseries/BaseTimeseriesServiceTest.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 672c17614c..2c5d8b9fea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -166,12 +166,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { } @Test - public void testFindByQueryOneMilisecondPeriod() throws Exception { + public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); saveEntries(deviceId, TS); + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); @@ -187,12 +188,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 10L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); @@ -208,12 +210,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 10L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -231,12 +234,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS-1); for (long i=TS; i<= TS + 8L; i++) { saveEntries(deviceId, i); } + saveEntries(deviceId, TS+1); - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -258,8 +262,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { saveEntries(deviceId, i); } - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); @@ -281,8 +284,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { saveEntries(deviceId, i); } - List queries = new ArrayList<>(); - queries.add(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); From cc67e93f972e1e1147fb4f4df1f93b6e77138670 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 29 Nov 2021 11:28:04 +0200 Subject: [PATCH 10/80] change tests cases for more understandable --- .../timeseries/BaseTimeseriesServiceTest.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 2c5d8b9fea..b6ca2d639c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -168,9 +168,9 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); + saveEntries(deviceId, TS - 1L); saveEntries(deviceId, TS); - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 1L); List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); @@ -188,115 +188,115 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 10, 11, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 5 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 5, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 10L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 100L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS-1); - for (long i=TS; i<= TS + 8L; i++) { + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 80L; i += 10L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS+1); + saveEntries(deviceId, TS + 80L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i=TS - 2; i<= TS + 12L; i++) { + for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 9, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 7 , new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i=TS - 2L; i<= TS + 12L; i++) { + for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10L) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 7, 5, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 2 , new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 6 , new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test From b54aa55c1ff6c82575b1604257fd1a185c0c7403 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 17 Jan 2022 12:49:36 +0200 Subject: [PATCH 11/80] update tests --- ...ctChunkedAggregationTimeseriesDaoTest.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index ae3d27fdf5..95cd2280fe 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,6 +19,8 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -59,7 +61,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3000, 2501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); @@ -69,7 +71,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -79,8 +81,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2998, 1500, LIMIT, COUNT, DESC); - ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2999, 3000, 3000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -92,7 +94,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -102,7 +104,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 0, 0, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); @@ -112,10 +114,11 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, query, 1, 2, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuery, 1, 2, COUNT); } @Test @@ -131,7 +134,7 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { @Test public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC); - ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1501, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); From d02c7ad7bf0346098bae1786142c053d1f75dc4c Mon Sep 17 00:00:00 2001 From: van-vanich Date: Mon, 17 Jan 2022 18:41:34 +0200 Subject: [PATCH 12/80] fix issue with license --- .../server/dao/timeseries/AggregationTimeseriesDao.java | 2 +- .../dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java index a99ad053b6..b8e42e3165 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 95cd2280fe..68fc8feccc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 51071a7413cf8a3d576af8627806be015869d363 Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 19 Jan 2022 18:06:53 +0200 Subject: [PATCH 13/80] remove unnecessary code and add some clean code --- .../server/dao/sqlts/SqlTimeseriesLatestDao.java | 1 - .../server/dao/sqlts/timescale/TimescaleTimeseriesDao.java | 7 ++++++- .../dao/service/timeseries/BaseTimeseriesServiceTest.java | 2 +- .../sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java | 4 ---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 6f58f318a1..7808f0e1bd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -46,7 +46,6 @@ import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index e15ee31b35..15875eb275 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -44,7 +44,12 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Function; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 7677654ec7..04c3425169 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -217,7 +217,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS + 50 , new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index 68fc8feccc..cab869a647 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -19,8 +19,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -47,8 +45,6 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { final String DESC = "DESC"; AbstractChunkedAggregationTimeseriesDao tsDao; - //SOME PRESENT: When we give data with period l-r, program return data in interval [l;r) - @Before public void setUp() throws Exception { tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); From 4642bf5893387d03613d62946b72d8ee1baa4af7 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Wed, 18 May 2022 18:17:02 +0300 Subject: [PATCH 14/80] coap-tests-spead-up --- .../server/transport/coap/CoapTestClient.java | 73 +++++++++++++++++++ ...AbstractCoapAttributesIntegrationTest.java | 64 ++++++++++++++++ .../CoapAttributesRequestIntegrationTest.java | 40 ---------- ...pAttributesRequestJsonIntegrationTest.java | 2 +- 4 files changed, 138 insertions(+), 41 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java new file mode 100644 index 0000000000..d746f4540d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java @@ -0,0 +1,73 @@ +package org.thingsboard.server.transport.coap; + +import org.eclipse.californium.core.CoapClient; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.MediaTypeRegistry; +import org.eclipse.californium.elements.exception.ConnectorException; +import org.thingsboard.server.common.msg.session.FeatureType; + +import java.io.IOException; + +public class CoapTestClient { + + private static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/"; + private static final long CLIENT_REQUEST_TIMEOUT = 60000L; + + private final CoapClient client; + + public CoapTestClient(){ + this.client = createClient(); + } + + public CoapTestClient(String accessToken, FeatureType featureType) { + this.client = createClient(getFeatureTokenUrl(accessToken, featureType)); + } + + public CoapTestClient(String featureTokenUrl) { + this.client = createClient(featureTokenUrl); + } + + public void connectToCoap(String accessToken, FeatureType featureType) { + connect(accessToken, featureType); + } + + public void connectToCoap(String accessToken) { + connect(accessToken, null); + } + + public void disconnect() { + if (client != null) { + client.shutdown(); + }; + } + + public CoapResponse PostMethod(String requestBody) throws ConnectorException, IOException { + return client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(requestBody.getBytes(), MediaTypeRegistry.APPLICATION_JSON); + } + + public CoapResponse GetMethod() throws ConnectorException, IOException { + return client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); + } + + private void connect(String accessToken, FeatureType featureType) { + if (client == null) { + throw new RuntimeException("Failed to connect! CoapClient is not initialized!"); + } + if (featureType == null){ + featureType = FeatureType.ATTRIBUTES; + } + client.setURI(getFeatureTokenUrl(accessToken, featureType)); + } + + private CoapClient createClient() { + return new CoapClient(); + } + + private CoapClient createClient(String featureTokenUrl) { + return new CoapClient(featureTokenUrl); + } + + public static String getFeatureTokenUrl(String token, FeatureType featureType) { + return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase(); + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index 9d0078aa3f..2ff733dc71 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -15,16 +15,31 @@ */ package org.thingsboard.server.transport.coap.attributes; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; +import org.eclipse.californium.core.coap.MediaTypeRegistry; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestClient; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + @Slf4j public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { + protected static final long CLIENT_REQUEST_TIMEOUT = 60000L; + protected static final String POST_ATTRIBUTES_PAYLOAD = "{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73," + "\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; @@ -49,4 +64,53 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap tsKvProtoBuilder.setKv(keyValueProto); return tsKvProtoBuilder.build(); } + + protected void processTestRequestAttributesValuesFromTheServer() throws Exception { + postAttributes(); + + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; + + List savedAttributeKeys = null; + while (start <= end) { + savedAttributeKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); + if (savedAttributeKeys.size() == 5) { + break; + } + Thread.sleep(100); + start += 100; + } + assertNotNull(savedAttributeKeys); + + String keys = "attribute1,attribute2,attribute3,attribute4,attribute5"; + //String featureTokenUrl = getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys; + //client = getCoapClient(featureTokenUrl); + + //CoapResponse getAttributesResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); + //validateResponse(getAttributesResponse); + + //TODO - NEW -- YURIY + String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys; + CoapTestClient client = new CoapTestClient(featureTokenUrl); + + validateResponse(client.GetMethod()); + } + + protected void postAttributes() throws Exception { + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + + //client = getCoapClient(FeatureType.ATTRIBUTES); + //CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON); + + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapResponse coapResponse = client.PostMethod(POST_ATTRIBUTES_PAYLOAD); + + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + } + + protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { + assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); + String expectedRequestPayload = "{\"client\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}},\"shared\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; + assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(getAttributesResponse.getPayload(), StandardCharsets.UTF_8))); + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java index 977502e0e6..b08a945c74 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java @@ -41,8 +41,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @DaoSqlTest public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributesIntegrationTest { - protected static final long CLIENT_REQUEST_TIMEOUT = 60000L; - @Before public void beforeTest() throws Exception { CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() @@ -60,42 +58,4 @@ public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributes public void testRequestAttributesValuesFromTheServer() throws Exception { processTestRequestAttributesValuesFromTheServer(); } - - protected void processTestRequestAttributesValuesFromTheServer() throws Exception { - postAttributes(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; - - List savedAttributeKeys = null; - while (start <= end) { - savedAttributeKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); - if (savedAttributeKeys.size() == 5) { - break; - } - Thread.sleep(100); - start += 100; - } - assertNotNull(savedAttributeKeys); - - String keys = "attribute1,attribute2,attribute3,attribute4,attribute5"; - String featureTokenUrl = getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys; - client = getCoapClient(featureTokenUrl); - - CoapResponse getAttributesResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); - validateResponse(getAttributesResponse); - } - - protected void postAttributes() throws Exception { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - client = getCoapClient(FeatureType.ATTRIBUTES); - CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - - protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { - assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); - String expectedRequestPayload = "{\"client\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}},\"shared\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; - assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(getAttributesResponse.getPayload(), StandardCharsets.UTF_8))); - } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java index e421fbf1a0..49928d6322 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java @@ -45,6 +45,6 @@ public class CoapAttributesRequestJsonIntegrationTest extends CoapAttributesRequ @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - super.testRequestAttributesValuesFromTheServer(); + processTestRequestAttributesValuesFromTheServer(); } } From 2d7e19b613a37a43d717afde36873931e1c4d87f Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Thu, 19 May 2022 17:57:08 +0300 Subject: [PATCH 15/80] coap-tests-spead-up --- .../server/transport/coap/CoapTestClient.java | 26 +- ...AbstractCoapAttributesIntegrationTest.java | 257 +++++++++++++++--- .../CoapAttributesRequestIntegrationTest.java | 2 +- ...pAttributesRequestJsonIntegrationTest.java | 2 +- ...AttributesRequestProtoIntegrationTest.java | 28 +- 5 files changed, 250 insertions(+), 65 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java index d746f4540d..5f1161acb1 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java @@ -27,12 +27,12 @@ public class CoapTestClient { this.client = createClient(featureTokenUrl); } - public void connectToCoap(String accessToken, FeatureType featureType) { - connect(accessToken, featureType); + public void connectToCoap(String accessToken) { + setURI(accessToken, null); } - public void connectToCoap(String accessToken) { - connect(accessToken, null); + public void connectToCoap(String accessToken, FeatureType featureType) { + setURI(accessToken, featureType); } public void disconnect() { @@ -41,22 +41,30 @@ public class CoapTestClient { }; } - public CoapResponse PostMethod(String requestBody) throws ConnectorException, IOException { - return client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(requestBody.getBytes(), MediaTypeRegistry.APPLICATION_JSON); + public CoapResponse postMethod(String requestBody) throws ConnectorException, IOException { + return this.postMethod(requestBody.getBytes()); } - public CoapResponse GetMethod() throws ConnectorException, IOException { + public CoapResponse postMethod(byte[] requestBodyBytes) throws ConnectorException, IOException { + return client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(requestBodyBytes, MediaTypeRegistry.APPLICATION_JSON); + } + + public CoapResponse getMethod() throws ConnectorException, IOException { return client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); } - private void connect(String accessToken, FeatureType featureType) { + public void setURI(String featureTokenUrl) { if (client == null) { throw new RuntimeException("Failed to connect! CoapClient is not initialized!"); } + client.setURI(featureTokenUrl); + } + + public void setURI(String accessToken, FeatureType featureType) { if (featureType == null){ featureType = FeatureType.ATTRIBUTES; } - client.setURI(getFeatureTokenUrl(accessToken, featureType)); + setURI(getFeatureTokenUrl(accessToken, featureType)); } private CoapClient createClient() { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index 2ff733dc71..7649f7bc91 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -16,24 +16,42 @@ package org.thingsboard.server.transport.coap.attributes; import com.fasterxml.jackson.core.type.TypeReference; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.device.profile.*; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.msg.session.FeatureType; +import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.transport.coap.CoapTestClient; +import org.thingsboard.server.transport.mqtt.MqttTestCallback; +import org.thingsboard.server.transport.mqtt.MqttTestClient; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.query.EntityKeyType.CLIENT_ATTRIBUTE; +import static org.thingsboard.server.common.data.query.EntityKeyType.SHARED_ATTRIBUTE; @Slf4j public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { @@ -43,12 +61,54 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap protected static final String POST_ATTRIBUTES_PAYLOAD = "{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73," + "\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; - protected List getTsKvProtoList() { - TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value1", TransportProtos.KeyValueType.STRING_V); - TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "true", TransportProtos.KeyValueType.BOOLEAN_V); - TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "42.0", TransportProtos.KeyValueType.DOUBLE_V); - TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "73", TransportProtos.KeyValueType.LONG_V); - TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); + public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + + "\n" + + "package test;\n" + + "\n" + + "message PostAttributes {\n" + + " string clientStr = 1;\n" + + " bool clientBool = 2;\n" + + " double clientDbl = 3;\n" + + " int32 clientLong = 4;\n" + + " JsonObject clientJson = 5;\n" + + "\n" + + " message JsonObject {\n" + + " int32 someNumber = 6;\n" + + " repeated int32 someArray = 7;\n" + + " NestedJsonObject someNestedObject = 8;\n" + + " message NestedJsonObject {\n" + + " string key = 9;\n" + + " }\n" + + " }\n" + + "}"; + + private static final String CLIENT_ATTRIBUTES_PAYLOAD = "{\"clientStr\":\"value1\",\"clientBool\":true,\"clientDbl\":42.0,\"clientLong\":73," + + "\"clientJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + + private static final String SHARED_ATTRIBUTES_PAYLOAD = "{\"sharedStr\":\"value1\",\"sharedBool\":true,\"sharedDbl\":42.0,\"sharedLong\":73," + + "\"sharedJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + + private List getTsKvProtoList(String attributePrefix) { + TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto(attributePrefix + "Str", "value1", TransportProtos.KeyValueType.STRING_V); + TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto(attributePrefix + "Bool", "true", TransportProtos.KeyValueType.BOOLEAN_V); + TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto(attributePrefix + "Dbl", "42.0", TransportProtos.KeyValueType.DOUBLE_V); + TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto(attributePrefix + "Long", "73", TransportProtos.KeyValueType.LONG_V); + TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto(attributePrefix + "Json", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); + List tsKvProtoList = new ArrayList<>(); + tsKvProtoList.add(tsKvProtoAttribute1); + tsKvProtoList.add(tsKvProtoAttribute2); + tsKvProtoList.add(tsKvProtoAttribute3); + tsKvProtoList.add(tsKvProtoAttribute4); + tsKvProtoList.add(tsKvProtoAttribute5); + return tsKvProtoList; + } + + public List getTsKvProtoList() { + TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("Str", "value1", TransportProtos.KeyValueType.STRING_V); + TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("Bool", "true", TransportProtos.KeyValueType.BOOLEAN_V); + TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("Dbl", "42.0", TransportProtos.KeyValueType.DOUBLE_V); + TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("Long", "73", TransportProtos.KeyValueType.LONG_V); + TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("Json", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); List tsKvProtoList = new ArrayList<>(); tsKvProtoList.add(tsKvProtoAttribute1); tsKvProtoList.add(tsKvProtoAttribute2); @@ -65,52 +125,169 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap return tsKvProtoBuilder.build(); } - protected void processTestRequestAttributesValuesFromTheServer() throws Exception { - postAttributes(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; + private List getEntityKeys(List keys, EntityKeyType scope) { + return keys.stream().map(key -> new EntityKey(scope, key)).collect(Collectors.toList()); + } - List savedAttributeKeys = null; - while (start <= end) { - savedAttributeKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); - if (savedAttributeKeys.size() == 5) { - break; - } - Thread.sleep(100); - start += 100; - } - assertNotNull(savedAttributeKeys); + private byte[] getAttributesProtoPayloadBytes() { - String keys = "attribute1,attribute2,attribute3,attribute4,attribute5"; - //String featureTokenUrl = getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys; - //client = getCoapClient(featureTokenUrl); + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); + DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - //CoapResponse getAttributesResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); - //validateResponse(getAttributesResponse); + DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); + Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); + assertNotNull(nestedJsonObjectBuilderDescriptor); + DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); - //TODO - NEW -- YURIY - String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys; - CoapTestClient client = new CoapTestClient(featureTokenUrl); + DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); + Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); + assertNotNull(jsonObjectBuilderDescriptor); + DynamicMessage jsonObject = jsonObjectBuilder + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) + .build(); - validateResponse(client.GetMethod()); + DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); + Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); + assertNotNull(postAttributesMsgDescriptor); + DynamicMessage postAttributesMsg = postAttributesBuilder + .setField(postAttributesMsgDescriptor.findFieldByName("clientStr"), "value1") + .setField(postAttributesMsgDescriptor.findFieldByName("clientBool"), true) + .setField(postAttributesMsgDescriptor.findFieldByName("clientDbl"), 42.0) + .setField(postAttributesMsgDescriptor.findFieldByName("clientLong"), 73) + .setField(postAttributesMsgDescriptor.findFieldByName("clientJson"), jsonObject) + .build(); + return postAttributesMsg.toByteArray(); } - protected void postAttributes() throws Exception { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + protected void processJsonTestRequestAttributesValuesFromTheServer() throws Exception { + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + SingleEntityFilter dtf = new SingleEntityFilter(); + dtf.setSingleEntity(savedDevice.getId()); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; + List clientKeysList = List.of(clientKeysStr.split(",")); + List sharedKeysList = List.of(sharedKeysStr.split(",")); + List csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE); + List shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE); + List keys = new ArrayList<>(); + keys.addAll(csKeys); + keys.addAll(shKeys); + getWsClient().subscribeLatestUpdate(keys, dtf); + getWsClient().registerWaitForUpdate(2); + + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", + SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - //client = getCoapClient(FeatureType.ATTRIBUTES); - //CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON); + CoapResponse coapResponse = client.postMethod(CLIENT_ATTRIBUTES_PAYLOAD); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + + String update = getWsClient().waitForUpdate(); + assertThat(update).as("ws update received").isNotBlank(); + String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; + client.setURI(featureTokenUrl); + validateJsonResponse(client.getMethod()); + client.disconnect(); + } + + protected void processProtoTestRequestAttributesValuesFromTheServer() throws Exception { CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); - CoapResponse coapResponse = client.PostMethod(POST_ATTRIBUTES_PAYLOAD); + SingleEntityFilter dtf = new SingleEntityFilter(); + dtf.setSingleEntity(savedDevice.getId()); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; + List clientKeysList = List.of(clientKeysStr.split(",")); + List sharedKeysList = List.of(sharedKeysStr.split(",")); + List csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE); + List shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE); + List keys = new ArrayList<>(); + keys.addAll(csKeys); + keys.addAll(shKeys); + getWsClient().subscribeLatestUpdate(keys, dtf); + getWsClient().registerWaitForUpdate(2); + + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", + SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + CoapResponse coapResponse = client.postMethod(getAttributesProtoPayloadBytes()); assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + + //validateProtoResponse(callback, getExpectedAttributeResponseMsg()); + +// client.publishAndWait(attrPubTopic, getAttributesProtoPayloadBytes()); +// client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); + String update = getWsClient().waitForUpdate(); + assertThat(update).as("ws update received").isNotBlank(); + + // MqttTestCallback callback = new MqttTestCallback(attrSubTopic.replace("+", "1")); +// client.setCallback(callback); + TransportApiProtos.AttributesRequest.Builder attributesRequestBuilder = TransportApiProtos.AttributesRequest.newBuilder(); + attributesRequestBuilder.setClientKeys(clientKeysStr); + attributesRequestBuilder.setSharedKeys(sharedKeysStr); + TransportApiProtos.AttributesRequest attributesRequest = attributesRequestBuilder.build(); +// client.publishAndWait(attrReqTopicPrefix + "1", attributesRequest.toByteArray()); + + String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; + client.setURI(featureTokenUrl); + + validateProtoResponse(client.getMethod()); + client.disconnect(); } - protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { + protected void postAttributes() throws Exception { +// //doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); +// +// doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", +// SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); +// +// +// +// //client = getCoapClient(FeatureType.ATTRIBUTES); +// //CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON); +// +// CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); +// CoapResponse coapResponse = client.postMethod(CLIENT_ATTRIBUTES_PAYLOAD); +// assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); +} + + protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); - String expectedRequestPayload = "{\"client\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}},\"shared\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; - assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(getAttributesResponse.getPayload(), StandardCharsets.UTF_8))); + String expectedResponse = "{\"client\":" + CLIENT_ATTRIBUTES_PAYLOAD + ",\"shared\":" + SHARED_ATTRIBUTES_PAYLOAD + "}"; + assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(getAttributesResponse.getPayload())); + } + + protected void validateProtoResponse(CoapResponse getAttributesResponse) throws InterruptedException, InvalidProtocolBufferException { + TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); + TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); + assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); + List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); + assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); + } + + private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { + TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); + List csTsKvProtoList = getTsKvProtoList("client"); + List shTsKvProtoList = getTsKvProtoList("shared"); + result.addAllClientAttributeList(csTsKvProtoList); + result.addAllSharedAttributeList(shTsKvProtoList); + result.setRequestId(0); + return result.build(); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java index b08a945c74..51d3968137 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java @@ -56,6 +56,6 @@ public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributes @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - processTestRequestAttributesValuesFromTheServer(); + processJsonTestRequestAttributesValuesFromTheServer(); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java index 49928d6322..2c6928b8d4 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java @@ -45,6 +45,6 @@ public class CoapAttributesRequestJsonIntegrationTest extends CoapAttributesRequ @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - processTestRequestAttributesValuesFromTheServer(); + processJsonTestRequestAttributesValuesFromTheServer(); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java index 7bdcce1400..85a5082e86 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java @@ -87,7 +87,7 @@ public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesReq @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - processTestRequestAttributesValuesFromTheServer(); + processProtoTestRequestAttributesValuesFromTheServer(); } protected void postAttributes() throws Exception { @@ -136,23 +136,23 @@ public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesReq assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); } - protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { - TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); - TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); - assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); - List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); - assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); + protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { +// TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); +// TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); +// assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); +// List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); +// List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); +// List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); +// List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); +// assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); +// assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); } private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); - List tsKvProtoList = getTsKvProtoList(); - result.addAllClientAttributeList(tsKvProtoList); - result.addAllSharedAttributeList(tsKvProtoList); + //List tsKvProtoList = getTsKvProtoList(); + //result.addAllClientAttributeList(tsKvProtoList); + //result.addAllSharedAttributeList(tsKvProtoList); result.setRequestId(0); return result.build(); } From 7ed34c16eb4ddca97a88d6150b43d5c053775092 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Wed, 25 May 2022 16:23:24 +0300 Subject: [PATCH 16/80] refactoring coap-tests-speed-up --- .../coap/AbstractCoapIntegrationTest.java | 17 -- .../transport/coap/CoapTestCallback.java | 53 +++++ .../server/transport/coap/CoapTestClient.java | 22 ++ ...AbstractCoapAttributesIntegrationTest.java | 220 ++++++++++++++---- .../CoapAttributesRequestIntegrationTest.java | 14 -- ...AttributesRequestProtoIntegrationTest.java | 114 --------- .../CoapAttributesUpdatesIntegrationTest.java | 170 +------------- ...pAttributesUpdatesJsonIntegrationTest.java | 7 +- ...AttributesUpdatesProtoIntegrationTest.java | 91 +------- .../coap/claim/CoapClaimDeviceTest.java | 10 +- .../coap/claim/CoapClaimProtoDeviceTest.java | 3 +- .../CoapProvisionJsonDeviceTest.java | 80 ++++--- .../CoapProvisionProtoDeviceTest.java | 57 ++--- ...tractCoapServerSideRpcIntegrationTest.java | 216 ++++++++++------- ...apServerSideRpcDefaultIntegrationTest.java | 7 +- .../CoapServerSideRpcJsonIntegrationTest.java | 4 +- ...CoapServerSideRpcProtoIntegrationTest.java | 87 +------ .../CoapAttributesIntegrationTest.java | 50 ++-- ...AbstractCoapTimeseriesIntegrationTest.java | 54 +++-- ...actCoapTimeseriesProtoIntegrationTest.java | 1 - 20 files changed, 516 insertions(+), 761 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index b3ab39fd51..4f77f53a32 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -55,12 +55,7 @@ public abstract class AbstractCoapIntegrationTest extends AbstractTransportInteg protected final byte[] EMPTY_PAYLOAD = new byte[0]; - protected CoapClient client; - protected void processAfterTest() throws Exception { - if (client != null) { - client.shutdown(); - } } protected void processBeforeTest(CoapTestConfigProperties config) throws Exception { @@ -156,16 +151,4 @@ public abstract class AbstractCoapIntegrationTest extends AbstractTransportInteg device.setType(type); return doPost("/api/device", device, Device.class); } - - protected CoapClient getCoapClient(FeatureType featureType) { - return new CoapClient(getFeatureTokenUrl(accessToken, featureType)); - } - - protected CoapClient getCoapClient(String featureTokenUrl) { - return new CoapClient(featureTokenUrl); - } - - protected String getFeatureTokenUrl(String token, FeatureType featureType) { - return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase(); - } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java new file mode 100644 index 0000000000..58379c9176 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java @@ -0,0 +1,53 @@ +package org.thingsboard.server.transport.coap; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.CoapHandler; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; + +import java.util.concurrent.CountDownLatch; + +@Slf4j +@Data +public class CoapTestCallback implements CoapHandler { + + protected final CountDownLatch latch; + protected Integer observe; + protected byte[] payloadBytes; + protected CoAP.ResponseCode responseCode; + + public CoapTestCallback() { + this.latch = new CountDownLatch(1); + } + + public CoapTestCallback(int subscribeCount) { + this.latch = new CountDownLatch(subscribeCount); + } + + public Integer getObserve() { + return observe; + } + + public byte[] getPayloadBytes() { + return payloadBytes; + } + + public CoAP.ResponseCode getResponseCode() { + return responseCode; + } + + @Override + public void onLoad(CoapResponse response) { + observe = response.getOptions().getObserve(); + payloadBytes = response.getPayload(); + responseCode = response.getCode(); + latch.countDown(); + } + + @Override + public void onError() { + log.warn("Command Response Ack Error, No connect"); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java index 5f1161acb1..54e0a8cfe9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java @@ -1,8 +1,12 @@ package org.thingsboard.server.transport.coap; import org.eclipse.californium.core.CoapClient; +import org.eclipse.californium.core.CoapHandler; +import org.eclipse.californium.core.CoapObserveRelation; import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.MediaTypeRegistry; +import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.elements.exception.ConnectorException; import org.thingsboard.server.common.msg.session.FeatureType; @@ -49,10 +53,24 @@ public class CoapTestClient { return client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(requestBodyBytes, MediaTypeRegistry.APPLICATION_JSON); } + public void postMethod(CoapHandler handler, String payload, int format) { + client.post(handler, payload, format); + } + + public void postMethod(CoapHandler handler, byte[] payload, int format) { + client.post(handler, payload, format); + } + public CoapResponse getMethod() throws ConnectorException, IOException { return client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); } + public CoapObserveRelation getObserveRelation(CoapTestCallback callback){ + Request request = Request.newGet().setObserve(); + request.setType(CoAP.Type.CON); + return client.observe(request, callback); + } + public void setURI(String featureTokenUrl) { if (client == null) { throw new RuntimeException("Failed to connect! CoapClient is not initialized!"); @@ -78,4 +96,8 @@ public class CoapTestClient { public static String getFeatureTokenUrl(String token, FeatureType featureType) { return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase(); } + + public static String getFeatureTokenUrl(String token, FeatureType featureType, int requestId) { + return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase() + "/" + requestId; + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index 7649f7bc91..b6f5613791 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -23,9 +23,12 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.squareup.wire.schema.internal.parser.ProtoFileElement; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.eclipse.californium.core.CoapObserveRelation; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.MediaTypeRegistry; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.device.profile.*; import org.thingsboard.server.common.data.query.DeviceTypeFilter; @@ -33,16 +36,20 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.msg.session.FeatureType; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestCallback; import org.thingsboard.server.transport.coap.CoapTestClient; +import org.thingsboard.server.transport.coap.attributes.updates.CoapAttributesUpdatesIntegrationTest; import org.thingsboard.server.transport.mqtt.MqttTestCallback; import org.thingsboard.server.transport.mqtt.MqttTestClient; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -56,10 +63,8 @@ import static org.thingsboard.server.common.data.query.EntityKeyType.SHARED_ATTR @Slf4j public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { - protected static final long CLIENT_REQUEST_TIMEOUT = 60000L; - - protected static final String POST_ATTRIBUTES_PAYLOAD = "{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73," + - "\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + @Autowired + DefaultTransportService defaultTransportService; public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + "\n" + @@ -88,7 +93,12 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap private static final String SHARED_ATTRIBUTES_PAYLOAD = "{\"sharedStr\":\"value1\",\"sharedBool\":true,\"sharedDbl\":42.0,\"sharedLong\":73," + "\"sharedJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; - private List getTsKvProtoList(String attributePrefix) { + protected static final String SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"sharedStr\":\"value\",\"sharedBool\":false,\"sharedDbl\":41.0,\"sharedLong\":72," + + "\"sharedJson\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}"; + + private static final String SHARED_ATTRIBUTES_DELETED_RESPONSE = "{\"deleted\":[\"sharedJson\"]}"; + + private List getTsKvProtoList(String attributePrefix) { TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto(attributePrefix + "Str", "value1", TransportProtos.KeyValueType.STRING_V); TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto(attributePrefix + "Bool", "true", TransportProtos.KeyValueType.BOOLEAN_V); TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto(attributePrefix + "Dbl", "42.0", TransportProtos.KeyValueType.DOUBLE_V); @@ -103,21 +113,6 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap return tsKvProtoList; } - public List getTsKvProtoList() { - TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("Str", "value1", TransportProtos.KeyValueType.STRING_V); - TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("Bool", "true", TransportProtos.KeyValueType.BOOLEAN_V); - TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("Dbl", "42.0", TransportProtos.KeyValueType.DOUBLE_V); - TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("Long", "73", TransportProtos.KeyValueType.LONG_V); - TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("Json", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); - List tsKvProtoList = new ArrayList<>(); - tsKvProtoList.add(tsKvProtoAttribute1); - tsKvProtoList.add(tsKvProtoAttribute2); - tsKvProtoList.add(tsKvProtoAttribute3); - tsKvProtoList.add(tsKvProtoAttribute4); - tsKvProtoList.add(tsKvProtoAttribute5); - return tsKvProtoList; - } - protected TransportProtos.TsKvProto getTsKvProto(String key, String value, TransportProtos.KeyValueType keyValueType) { TransportProtos.TsKvProto.Builder tsKvProtoBuilder = TransportProtos.TsKvProto.newBuilder(); TransportProtos.KeyValueProto keyValueProto = getKeyValueProto(key, value, keyValueType); @@ -225,43 +220,86 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap CoapResponse coapResponse = client.postMethod(getAttributesProtoPayloadBytes()); assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - //validateProtoResponse(callback, getExpectedAttributeResponseMsg()); - -// client.publishAndWait(attrPubTopic, getAttributesProtoPayloadBytes()); -// client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); String update = getWsClient().waitForUpdate(); assertThat(update).as("ws update received").isNotBlank(); - // MqttTestCallback callback = new MqttTestCallback(attrSubTopic.replace("+", "1")); -// client.setCallback(callback); - TransportApiProtos.AttributesRequest.Builder attributesRequestBuilder = TransportApiProtos.AttributesRequest.newBuilder(); - attributesRequestBuilder.setClientKeys(clientKeysStr); - attributesRequestBuilder.setSharedKeys(sharedKeysStr); - TransportApiProtos.AttributesRequest attributesRequest = attributesRequestBuilder.build(); -// client.publishAndWait(attrReqTopicPrefix + "1", attributesRequest.toByteArray()); - String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; client.setURI(featureTokenUrl); - validateProtoResponse(client.getMethod()); client.disconnect(); } - protected void postAttributes() throws Exception { -// //doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); -// -// doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", -// SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); -// -// -// -// //client = getCoapClient(FeatureType.ATTRIBUTES); -// //CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON); -// -// CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); -// CoapResponse coapResponse = client.postMethod(CLIENT_ATTRIBUTES_PAYLOAD); -// assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); -} + protected void processJsonTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { + if (!emptyCurrentStateNotification) { + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); + } + + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapTestCallback callbackCoap = new CoapTestCallback(1); + + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + + if (emptyCurrentStateNotification) { + validateUpdateAttributesJsonResponse(callbackCoap, "{}", 0); + } else { + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, 0); + } + + CountDownLatch latch = new CountDownLatch(1); + int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1; + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + latch.await(3, TimeUnit.SECONDS); + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD, expectedObserveCnt); + + latch = new CountDownLatch(1); + int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1; + doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); + latch.await(3, TimeUnit.SECONDS); + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_DELETED_RESPONSE, expectedObserveBeforeDeleteCnt); + + observeRelation.proactiveCancel(); + assertTrue(observeRelation.isCanceled()); + + awaitClientAfterCancelObserve(); + client.disconnect(); + } + + protected void processProtoTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { + if (!emptyCurrentStateNotification) { + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); + } + + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapTestCallback callbackCoap = new CoapTestCallback(1); + + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + + if (emptyCurrentStateNotification) { + validateEmptyCurrentStateAttributesProtoResponse(callbackCoap); + } else { + validateCurrentStateAttributesProtoResponse(callbackCoap); + } + + CountDownLatch latch = new CountDownLatch(1); + int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1; + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + latch.await(3, TimeUnit.SECONDS); + validateUpdateProtoAttributesResponse(callbackCoap, expectedObserveCnt); + + latch = new CountDownLatch(1); + int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1; + doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); + latch.await(3, TimeUnit.SECONDS); + validateDeleteProtoAttributesResponse(callbackCoap, expectedObserveBeforeDeleteCnt); + + observeRelation.proactiveCancel(); + assertTrue(observeRelation.isCanceled()); + + awaitClientAfterCancelObserve(); + client.disconnect(); + } protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); @@ -281,6 +319,92 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); } + protected void validateUpdateAttributesJsonResponse(CoapTestCallback callback, String expectedResponse, int expectedObserveCnt) { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); + assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.toJsonNode(response)); + } + + protected void validateEmptyCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException { + assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(0, callback.getObserve().intValue()); + } + + protected void validateCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(0, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("sharedStr", "value", TransportProtos.KeyValueType.STRING_V); + TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("sharedBool", "false", TransportProtos.KeyValueType.BOOLEAN_V); + TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("sharedDbl", "41.0", TransportProtos.KeyValueType.DOUBLE_V); + TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("sharedLong", "72", TransportProtos.KeyValueType.LONG_V); + TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("sharedJson", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); + List tsKvProtoList = new ArrayList<>(); + tsKvProtoList.add(tsKvProtoAttribute1); + tsKvProtoList.add(tsKvProtoAttribute2); + tsKvProtoList.add(tsKvProtoAttribute3); + tsKvProtoList.add(tsKvProtoAttribute4); + tsKvProtoList.add(tsKvProtoAttribute5); + TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build(); + TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + List expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + + assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); + assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); + } + + protected void validateUpdateProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + List tsKvProtoList = getTsKvProtoList("shared"); + attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); + + TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); + TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + List actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + + assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); + assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); + } + + protected void validateDeleteProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); + + TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); + TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size()); + assertEquals("sharedJson", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0)); + } + + private void awaitClientAfterCancelObserve() { + Awaitility.await("awaitClientAfterCancelObserve") + .pollInterval(10, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .until(()->{ + log.trace("awaiting defaultTransportService.sessions is empty"); + return defaultTransportService.sessions.isEmpty();}); + } + private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); List csTsKvProtoList = getTsKvProtoList("client"); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java index 51d3968137..727fb52380 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java @@ -15,27 +15,13 @@ */ package org.thingsboard.server.transport.coap.attributes.request; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; -import org.thingsboard.server.common.msg.session.FeatureType; - -import java.nio.charset.StandardCharsets; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j @DaoSqlTest diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java index 85a5082e86..5f300ba68e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java @@ -15,64 +15,18 @@ */ package org.thingsboard.server.transport.coap.attributes.request; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @Slf4j @DaoSqlTest public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesRequestIntegrationTest { - public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + - "\n" + - "package test;\n" + - "\n" + - "message PostAttributes {\n" + - " string attribute1 = 1;\n" + - " bool attribute2 = 2;\n" + - " double attribute3 = 3;\n" + - " int32 attribute4 = 4;\n" + - " JsonObject attribute5 = 5;\n" + - "\n" + - " message JsonObject {\n" + - " int32 someNumber = 6;\n" + - " repeated int32 someArray = 7;\n" + - " NestedJsonObject someNestedObject = 8;\n" + - " message NestedJsonObject {\n" + - " string key = 9;\n" + - " }\n" + - " }\n" + - "}"; - @Before @Override public void beforeTest() throws Exception { @@ -89,72 +43,4 @@ public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesReq public void testRequestAttributesValuesFromTheServer() throws Exception { processProtoTestRequestAttributesValuesFromTheServer(); } - - protected void postAttributes() throws Exception { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - - DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); - Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); - assertNotNull(nestedJsonObjectBuilderDescriptor); - DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); - - DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); - Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); - assertNotNull(jsonObjectBuilderDescriptor); - DynamicMessage jsonObject = jsonObjectBuilder - .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) - .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) - .build(); - - DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); - Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); - assertNotNull(postAttributesMsgDescriptor); - DynamicMessage postAttributesMsg = postAttributesBuilder - .setField(postAttributesMsgDescriptor.findFieldByName("attribute1"), "value1") - .setField(postAttributesMsgDescriptor.findFieldByName("attribute2"), true) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute3"), 42.0) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute4"), 73) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute5"), jsonObject) - .build(); - byte[] payload = postAttributesMsg.toByteArray(); - client = getCoapClient(FeatureType.ATTRIBUTES); - CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - - protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { -// TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); -// TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); -// assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); -// List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); -// List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); -// List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); -// List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); -// assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); -// assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); - } - - private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { - TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); - //List tsKvProtoList = getTsKvProtoList(); - //result.addAllClientAttributeList(tsKvProtoList); - //result.addAllSharedAttributeList(tsKvProtoList); - result.setRequestId(0); - return result.build(); - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java index bbbfad3c65..343ee15afc 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java @@ -15,61 +15,20 @@ */ package org.thingsboard.server.transport.coap.attributes.updates; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.awaitility.Awaitility; -import org.eclipse.californium.core.CoapHandler; -import org.eclipse.californium.core.CoapObserveRelation; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.Request; -import org.eclipse.californium.core.server.resources.Resource; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.coapserver.DefaultCoapServerService; -import org.thingsboard.server.common.msg.session.FeatureType; -import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import org.thingsboard.server.transport.coap.CoapTransportResource; import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.spy; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @Slf4j @DaoSqlTest public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributesIntegrationTest { - private static final String RESPONSE_ATTRIBUTES_PAYLOAD_DELETED = "{\"deleted\":[\"attribute5\"]}"; - - protected static final String POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"attribute1\":\"value\",\"attribute2\":false,\"attribute3\":41.0,\"attribute4\":72," + - "\"attribute5\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}"; - - CoapTransportResource coapTransportResource; - - @Autowired - DefaultCoapServerService defaultCoapServerService; - - @Autowired - DefaultTransportService defaultTransportService; - @Before public void beforeTest() throws Exception { - Resource api = defaultCoapServerService.getCoapServer().getRoot().getChild("api"); - coapTransportResource = spy( (CoapTransportResource) api.getChild("v1") ); - api.delete(api.getChild("v1") ); - api.add(coapTransportResource); CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() .deviceName("Test Subscribe to attribute updates") .build(); @@ -83,136 +42,11 @@ public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributes @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - processTestSubscribeToAttributesUpdates(false); + processJsonTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - processTestSubscribeToAttributesUpdates(true); - } - - protected void processTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { - if (!emptyCurrentStateNotification) { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); - } - client = getCoapClient(FeatureType.ATTRIBUTES); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(latch); - - Request request = Request.newGet().setObserve(); - request.setType(CoAP.Type.CON); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); - - if (emptyCurrentStateNotification) { - validateEmptyCurrentStateAttributesResponse(callback); - } else { - validateCurrentStateAttributesResponse(callback); - } - - latch = new CountDownLatch(1); - int expectedObserveCnt = callback.getObserve().intValue() + 1; - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateUpdateAttributesResponse(callback, expectedObserveCnt); - - - latch = new CountDownLatch(1); - int expectedObserveBeforeDeleteCnt = callback.getObserve().intValue() + 1; - doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=attribute5", String.class); - latch.await(3, TimeUnit.SECONDS); - - validateDeleteAttributesResponse(callback, expectedObserveBeforeDeleteCnt); - observeRelation.proactiveCancel(); - assertTrue(observeRelation.isCanceled()); - - awaitClientAfterCancelObserve(); - } - - protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION), JacksonUtil.toJsonNode(response)); - } - - protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals("{}", response); - } - - protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD), JacksonUtil.toJsonNode(response)); - } - - protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(RESPONSE_ATTRIBUTES_PAYLOAD_DELETED), JacksonUtil.toJsonNode(response)); - } - - protected static class TestCoapCallback implements CoapHandler { - - private final CountDownLatch latch; - - private Integer observe; - private byte[] payloadBytes; - private CoAP.ResponseCode responseCode; - - public Integer getObserve() { - return observe; - } - - public byte[] getPayloadBytes() { - return payloadBytes; - } - - public CoAP.ResponseCode getResponseCode() { - return responseCode; - } - - private TestCoapCallback(CountDownLatch latch) { - this.latch = latch; - } - - @Override - public void onLoad(CoapResponse response) { - observe = response.getOptions().getObserve(); - payloadBytes = response.getPayload(); - responseCode = response.getCode(); - latch.countDown(); - } - - @Override - public void onError() { - log.warn("Command Response Ack Error, No connect"); - } - - } - - private void awaitClientAfterCancelObserve() { - Awaitility.await("awaitClientAfterCancelObserve") - .pollInterval(10, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .until(()->{ - log.trace("awaiting defaultTransportService.sessions is empty"); - return defaultTransportService.sessions.isEmpty();}); + processJsonTestSubscribeToAttributesUpdates(true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java index 4ab052db46..816ffa53b3 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java @@ -23,10 +23,11 @@ import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; +import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; @Slf4j @DaoSqlTest -public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpdatesIntegrationTest { +public class CoapAttributesUpdatesJsonIntegrationTest extends AbstractCoapAttributesIntegrationTest { @Before public void beforeTest() throws Exception { @@ -45,11 +46,11 @@ public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpda @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - super.testSubscribeToAttributesUpdatesFromTheServer(); + processJsonTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - super.testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification(); + processJsonTestSubscribeToAttributesUpdates(true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java index 4d9970b2dc..4a5e7c033f 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java @@ -15,30 +15,19 @@ */ package org.thingsboard.server.transport.coap.attributes.updates; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.coap.CoAP; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertTrue; +import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; @Slf4j @DaoSqlTest -public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpdatesIntegrationTest { +public class CoapAttributesUpdatesProtoIntegrationTest extends AbstractCoapAttributesIntegrationTest { @Before public void beforeTest() throws Exception { @@ -57,83 +46,11 @@ public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpd @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - processTestSubscribeToAttributesUpdates(false); + processProtoTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - processTestSubscribeToAttributesUpdates(true); + processProtoTestSubscribeToAttributesUpdates(true); } - - protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value", TransportProtos.KeyValueType.STRING_V); - TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "false", TransportProtos.KeyValueType.BOOLEAN_V); - TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "41.0", TransportProtos.KeyValueType.DOUBLE_V); - TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "72", TransportProtos.KeyValueType.LONG_V); - TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); - List tsKvProtoList = new ArrayList<>(); - tsKvProtoList.add(tsKvProtoAttribute1); - tsKvProtoList.add(tsKvProtoAttribute2); - tsKvProtoList.add(tsKvProtoAttribute3); - tsKvProtoList.add(tsKvProtoAttribute4); - tsKvProtoList.add(tsKvProtoAttribute5); - TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build(); - TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - List expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - - assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); - assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); - - } - - protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - } - - protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - List tsKvProtoList = getTsKvProtoList(); - attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); - - TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); - TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - List actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - - assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); - assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); - - } - - protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - attributeUpdateNotificationMsgBuilder.addSharedDeleted("attribute5"); - - TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); - TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size()); - assertEquals("attribute5", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0)); - - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java index de9c024bb8..1e2c65ff35 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java @@ -34,6 +34,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import java.io.IOException; @@ -96,7 +97,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { log.warn("[testClaimingDevice] Device: {}, Transport type: {}", savedDevice.getName(), savedDevice.getType()); - client = getCoapClient(FeatureType.CLAIM); + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; byte[] failurePayloadBytes; if (emptyPayload) { @@ -109,7 +110,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { validateClaimResponse(emptyPayload, client, payloadBytes, failurePayloadBytes); } - protected void validateClaimResponse(boolean emptyPayload, CoapClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception { + protected void validateClaimResponse(boolean emptyPayload, CoapTestClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception { postClaimRequest(client, failurePayloadBytes); loginUser(customerAdmin.getName(), CUSTOMER_USER_PASSWORD); @@ -143,10 +144,11 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { claimResponse = doPostClaimAsync("/api/customer/device/" + savedDevice.getName() + "/claim", claimRequest, ClaimResponse.class, status().isBadRequest()); assertEquals(claimResponse, ClaimResponse.CLAIMED); + client.disconnect(); } - private void postClaimRequest(CoapClient client, byte[] payload) throws IOException, ConnectorException { - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + private void postClaimRequest(CoapTestClient client, byte[] payload) throws IOException, ConnectorException { + CoapResponse coapResponse = client.postMethod(payload); assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java index 812ea7392b..e4090cd07a 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportApiProtos; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; @Slf4j @@ -56,7 +57,7 @@ public class CoapClaimProtoDeviceTest extends CoapClaimDeviceTest { @Override protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { - client = getCoapClient(FeatureType.CLAIM); + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; if (emptyPayload) { TransportApiProtos.ClaimDevice claimDevice = getClaimDevice(0, emptyPayload); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java index a952ef6d63..ded7b3a65b 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java @@ -15,16 +15,13 @@ */ package org.thingsboard.server.transport.coap.provision; -import com.google.gson.JsonObject; +import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.common.data.CoapDeviceType; @@ -34,14 +31,12 @@ import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.session.FeatureType; -import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; - @Slf4j @DaoSqlTest public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { @@ -95,10 +90,11 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .transportPayloadType(TransportPayloadType.JSON) .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); - Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("errorMsg")); + Assert.assertTrue(response.hasNonNull("status")); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText()); } @@ -112,8 +108,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -121,8 +118,8 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } @@ -137,8 +134,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .build(); processBeforeTest(configProperties); String requestCredentials = ",\"credentialsType\": \"ACCESS_TOKEN\",\"token\": \"test_token\""; - byte[] result = createCoapClientAndPublish(requestCredentials).getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials)); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -146,10 +144,10 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "ACCESS_TOKEN"); Assert.assertEquals(deviceCredentials.getCredentialsId(), "test_token"); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } @@ -164,8 +162,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .build(); processBeforeTest(configProperties); String requestCredentials = ",\"credentialsType\": \"X509_CERTIFICATE\",\"hash\": \"testHash\""; - byte[] result = createCoapClientAndPublish(requestCredentials).getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials)); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -173,7 +172,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "X509_CERTIFICATE"); String cert = EncryptionUtil.certTrimNewLines(deviceCredentials.getCredentialsValue()); @@ -182,7 +181,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { Assert.assertEquals(deviceCredentials.getCredentialsId(), sha3Hash); Assert.assertEquals(deviceCredentials.getCredentialsValue(), "testHash"); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } private void processTestProvisioningCheckPreProvisionedDevice() throws Exception { @@ -195,13 +194,14 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } private void processTestProvisioningWithBadKeyDevice() throws Exception { @@ -214,25 +214,23 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); - Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("errorMsg")); + Assert.assertTrue(response.hasNonNull("status")); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText()); } - private CoapResponse createCoapClientAndPublish() throws Exception { + private byte[] createCoapClientAndPublish() throws Exception { return createCoapClientAndPublish(""); } - private CoapResponse createCoapClientAndPublish(String deviceCredentials) throws Exception { + private byte[] createCoapClientAndPublish(String deviceCredentials) throws Exception { String provisionRequestMsg = createTestProvisionMessage(deviceCredentials); - client = getCoapClient(FeatureType.PROVISION); - return postProvision(client, provisionRequestMsg.getBytes()); - } - - - private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException { - return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.PROVISION); + byte[] responseByte = client.postMethod(provisionRequestMsg.getBytes()).getPayload(); + client.disconnect(); + return responseByte; } private String createTestProvisionMessage(String deviceCredentials) { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java index a2e270796e..74e39963a5 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceReque import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import java.io.IOException; @@ -101,9 +102,9 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .transportPayloadType(TransportPayloadType.PROTOBUF) .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); Assert.assertNotNull(result); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().toString()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().name()); } private void processTestProvisioningCreateNewDeviceWithoutCredentials() throws Exception { @@ -116,7 +117,7 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -124,8 +125,8 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name()); } private void processTestProvisioningCreateNewDeviceWithAccessToken() throws Exception { @@ -138,9 +139,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()).build(); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder() + .setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()) + .build(); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials)).getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom( + createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials))); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -164,9 +168,13 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceX509CertRequestMsg(ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()).build(); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder() + .setValidateDeviceX509CertRequestMsg( + ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()) + .build(); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials)).getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom( + createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials))); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -196,12 +204,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name()); } private void processTestProvisioningWithBadKeyDevice() throws Exception { @@ -214,24 +222,20 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().toString()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().name()); } - private CoapResponse createCoapClientAndPublish() throws Exception { - byte[] provisionRequestMsg = createTestProvisionMessage(); - CoapResponse coapResponse = createCoapClientAndPublish(provisionRequestMsg); - Assert.assertNotNull("COAP response", coapResponse); - return coapResponse; - } - - private CoapResponse createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception { - client = getCoapClient(FeatureType.PROVISION); - return postProvision(client, provisionRequestMsg); + private byte[] createCoapClientAndPublish() throws Exception { + return createCoapClientAndPublish(createTestProvisionMessage()); } - private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException { - return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + private byte[] createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception { + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.PROVISION); + CoapResponse coapResponse = client.postMethod(provisionRequestMsg); + Assert.assertNotNull("COAP response", coapResponse); + client.disconnect(); + return coapResponse.getPayload(); } private byte[] createTestsProvisionMessage(CredentialsType credentialsType, CredentialsDataProto credentialsData) throws Exception { @@ -247,7 +251,6 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .toByteArray(); } - private byte[] createTestProvisionMessage() throws Exception { return createTestsProvisionMessage(null, null); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java index 5262b55b97..36cd372b1f 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java @@ -16,18 +16,29 @@ package org.thingsboard.server.transport.coap.rpc; import com.fasterxml.jackson.databind.JsonNode; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapHandler; import org.eclipse.californium.core.CoapObserveRelation; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.core.coap.Request; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestCallback; +import org.thingsboard.server.transport.coap.CoapTestClient; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -41,76 +52,76 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractCoapIntegrationTest { + public static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" + + "package rpc;\n" + + "\n" + + "message RpcRequestMsg {\n" + + " optional string method = 1;\n" + + " optional int32 requestId = 2;\n" + + " Params params = 3;\n" + + "\n" + + " message Params {\n" + + " optional string pin = 1;\n" + + " optional int32 value = 2;\n" + + " }\n" + + "}"; + protected static final String DEVICE_RESPONSE = "{\"value1\":\"A\",\"value2\":\"B\"}"; protected static final Long asyncContextTimeoutToUseRpcPlugin = 10000L; - protected void processOneWayRpcTest() throws Exception { - client = getCoapClient(FeatureType.RPC); - client.useCONs(); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(client, latch, true); + protected void processOneWayRpcTest(boolean protobuf) throws Exception { + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.RPC); + CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, true, protobuf); - Request request = Request.newGet().setObserve(); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); - - validateCurrentStateNotification(callback); - - latch = new CountDownLatch(1); + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + validateCurrentStateNotification(callbackCoap); + CountDownLatch latch = new CountDownLatch(1); String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateOneWayStateChangedNotification(callback, result); + validateOneWayStateChangedNotification(callbackCoap, result); observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); + client.disconnect(); } - protected void processTwoWayRpcTest(String expectedResponseResult) throws Exception { - client = getCoapClient(FeatureType.RPC); - client.useCONs(); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(client, latch, false); + protected void processTwoWayRpcTest(String expectedResponseResult, boolean protobuf) throws Exception { + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.RPC); + CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, false, protobuf); - Request request = Request.newGet().setObserve(); - request.setType(CoAP.Type.CON); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - validateCurrentStateNotification(callback); + validateCurrentStateNotification(callbackCoap); String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateTwoWayStateChangedNotification(callback, 1, expectedResponseResult, actualResult); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - latch = new CountDownLatch(1); + validateTwoWayStateChangedNotification(callbackCoap, 1, expectedResponseResult, actualResult); + CountDownLatch latch = new CountDownLatch(1); actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - validateTwoWayStateChangedNotification(callback, 2, expectedResponseResult, actualResult); + validateTwoWayStateChangedNotification(callbackCoap, 2, expectedResponseResult, actualResult); observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); + client.disconnect(); } - protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { + protected void processOnLoadResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { JsonNode responseJson = JacksonUtil.fromBytes(response.getPayload()); - client.setURI(getRpcResponseFeatureTokenUrl(accessToken, responseJson.get("id").asInt())); - client.post(new CoapHandler() { + client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, responseJson.get("id").asInt())); + client.postMethod(new CoapHandler() { @Override public void onLoad(CoapResponse response) { log.warn("Command Response Ack: {}, {}", response.getCode(), response.getResponseText()); @@ -124,36 +135,89 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC }, DEVICE_RESPONSE, MediaTypeRegistry.APPLICATION_JSON); } - protected String getRpcResponseFeatureTokenUrl(String token, int requestId) { - return COAP_BASE_URL + token + "/" + FeatureType.RPC.name().toLowerCase() + "/" + requestId; + protected void processOnLoadProtoResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); + ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); + DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + + byte[] requestPayload = response.getPayload(); + DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); + Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType(); + try { + DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); + Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); + int requestId = (int) dynamicMessage.getField(requestIdDescriptor); + ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); + DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); + Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); + DynamicMessage rpcResponseMsg = rpcResponseBuilder + .setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE) + .build(); + client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, requestId)); + client.postMethod(new CoapHandler() { + @Override + public void onLoad(CoapResponse response) { + log.warn("Command Response Ack: {}", response.getCode()); + latch.countDown(); + } + + @Override + public void onError() { + log.warn("Command Response Ack Error, No connect"); + } + }, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON); + } catch (InvalidProtocolBufferException e) { + log.warn("Command Response Ack Error, Invalid response received: ", e); + } } - protected class TestCoapCallback implements CoapHandler { + private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() { + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + } - private final CoapClient client; - private final CountDownLatch latch; - private final boolean isOneWayRpc; + private void validateCurrentStateNotification(CoapTestCallback callback) { + assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID); + assertEquals(0, callback.getObserve().intValue()); + } - private Integer observe; - private byte[] payloadBytes; - private CoAP.ResponseCode responseCode; + private void validateOneWayStateChangedNotification(CoapTestCallback callback, String result) { + assertTrue(StringUtils.isEmpty(result)); + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(1, callback.getObserve().intValue()); + } - public Integer getObserve() { - return observe; - } + private void validateTwoWayStateChangedNotification(CoapTestCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) { + assertEquals(expectedResult, actualResult); + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveNumber, callback.getObserve().intValue()); + } - public byte[] getPayloadBytes() { - return payloadBytes; - } + protected class TestCoapCallbackForRPC extends CoapTestCallback { - public CoAP.ResponseCode getResponseCode() { - return responseCode; - } + private final CoapTestClient client; + private final boolean isOneWayRpc; + private final boolean protobuf; - TestCoapCallback(CoapClient client, CountDownLatch latch, boolean isOneWayRpc) { + TestCoapCallbackForRPC(CoapTestClient client, int subscribeCount, boolean isOneWayRpc, boolean protobuf) { + super(subscribeCount); this.client = client; - this.latch = latch; this.isOneWayRpc = isOneWayRpc; + this.protobuf = protobuf; } @Override @@ -163,7 +227,11 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC observe = response.getOptions().getObserve(); if (observe != null) { if (!isOneWayRpc && observe > 0) { - processOnLoadResponse(response, client, observe, latch); + if (!protobuf){ + processOnLoadResponse(response, client, observe, latch); + } else { + processOnLoadProtoResponse(response, client, observe, latch); + } } else { latch.countDown(); } @@ -174,31 +242,5 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC public void onError() { log.warn("Command Response Ack Error, No connect"); } - - } - - private void validateCurrentStateNotification(TestCoapCallback callback) { - assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID); - assertEquals(0, callback.getObserve().intValue()); } - - private void validateOneWayStateChangedNotification(TestCoapCallback callback, String result) { - assertTrue(StringUtils.isEmpty(result)); - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(1, callback.getObserve().intValue()); - } - - private void validateTwoWayStateChangedNotification(TestCoapCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) { - assertEquals(expectedResult, actualResult); - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveNumber, callback.getObserve().intValue()); - } - - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java index cd0e9268b4..d2ef37f795 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java @@ -21,9 +21,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; -import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; @@ -87,12 +84,12 @@ public class CoapServerSideRpcDefaultIntegrationTest extends AbstractCoapServerS @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(false); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}"); + processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java index 6dde4b919b..96ab02325a 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java @@ -45,12 +45,12 @@ public class CoapServerSideRpcJsonIntegrationTest extends AbstractCoapServerSide @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(false); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}"); + processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java index ae1c89d765..93b5c6c6fb 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java @@ -15,52 +15,19 @@ */ package org.thingsboard.server.transport.coap.rpc; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.eclipse.californium.core.CoapHandler; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.util.concurrent.CountDownLatch; - -import static org.junit.Assert.assertTrue; - @Slf4j @DaoSqlTest public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSideRpcIntegrationTest { - private static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" + - "package rpc;\n" + - "\n" + - "message RpcRequestMsg {\n" + - " optional string method = 1;\n" + - " optional int32 requestId = 2;\n" + - " Params params = 3;\n" + - "\n" + - " message Params {\n" + - " optional string pin = 1;\n" + - " optional int32 value = 2;\n" + - " }\n" + - "}"; - @Before public void beforeTest() throws Exception { CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() @@ -79,61 +46,11 @@ public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSid @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(true); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}"); - } - - @Override - protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); - - byte[] requestPayload = response.getPayload(); - DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); - Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType(); - try { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); - Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); - int requestId = (int) dynamicMessage.getField(requestIdDescriptor); - ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); - DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); - Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); - DynamicMessage rpcResponseMsg = rpcResponseBuilder - .setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE) - .build(); - client.setURI(getRpcResponseFeatureTokenUrl(accessToken, requestId)); - client.post(new CoapHandler() { - @Override - public void onLoad(CoapResponse response) { - log.warn("Command Response Ack: {}", response.getCode()); - latch.countDown(); - } - - @Override - public void onError() { - log.warn("Command Response Ack Error, No connect"); - } - }, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON); - } catch (InvalidProtocolBufferException e) { - log.warn("Command Response Ack Error, Invalid response received: ", e); - } - } - - private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}", true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java index ee566be2bb..bd8c9a47f6 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java @@ -17,11 +17,8 @@ package org.thingsboard.server.transport.coap.telemetry.attributes; import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -29,9 +26,9 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.msg.session.FeatureType; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedHashMap; @@ -74,30 +71,17 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { } protected void processAttributesTest(List expectedKeys, byte[] payload, boolean presenceFieldsTest) throws Exception { - client = getCoapClient(FeatureType.ATTRIBUTES); - postAttributes(client, payload); + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapResponse coapResponse = client.postMethod(payload); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); DeviceId deviceId = savedDevice.getId(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; - - List actualKeys = null; - while (start <= end) { - actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); - if (actualKeys.size() == expectedKeys.size()) { - break; - } - Thread.sleep(100); - start += 100; - } + List actualKeys = getActualKeysList(deviceId, expectedKeys); assertNotNull(actualKeys); Set actualKeySet = new HashSet<>(actualKeys); - Set expectedKeySet = new HashSet<>(expectedKeys); - assertEquals(expectedKeySet, actualKeySet); String getAttributesValuesUrl = getAttributesValuesUrl(deviceId, actualKeySet); @@ -111,14 +95,6 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { doDelete(deleteAttributesUrl); } - private void postAttributes(CoapClient client, byte[] payload) throws IOException, ConnectorException { - if (payload == null) { - payload = PAYLOAD_VALUES_STR.getBytes(); - } - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - @SuppressWarnings({"unchecked", "rawtypes"}) protected void assertAttributesValues(List> deviceValues, Set keySet) { for (Map map : deviceValues) { @@ -171,6 +147,22 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { } } + private List getActualKeysList(DeviceId deviceId, List expectedKeys) throws Exception { + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; + + List actualKeys = null; + while (start <= end) { + actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); + if (actualKeys.size() == expectedKeys.size()) { + break; + } + Thread.sleep(100); + start += 100; + } + return actualKeys; + } + private String getAttributesValuesUrl(DeviceId deviceId, Set actualKeySet) { return "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/attributes/CLIENT_SCOPE?keys=" + String.join(",", actualKeySet); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java index 0c7ab4d1c5..31c60f49f6 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java @@ -17,19 +17,17 @@ package org.thingsboard.server.transport.coap.telemetry.timeseries; import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -75,28 +73,16 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } protected void processTestPostTelemetry(byte[] payloadBytes, List expectedKeys, boolean withTs, boolean presenceFieldsTest) throws Exception { - client = getCoapClient(FeatureType.TELEMETRY); - postTelemetry(client, payloadBytes); - - String deviceId = savedDevice.getId().getId().toString(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; + CoapTestClient client = new CoapTestClient(accessToken, FeatureType.TELEMETRY); + CoapResponse coapResponse = client.postMethod(payloadBytes); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - List actualKeys = null; - while (start <= end) { - actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {}); - if (actualKeys.size() == expectedKeys.size()) { - break; - } - Thread.sleep(100); - start += 100; - } + DeviceId deviceId = savedDevice.getId(); + List actualKeys = getActualKeysList(deviceId, expectedKeys); assertNotNull(actualKeys); Set actualKeySet = new HashSet<>(actualKeys); Set expectedKeySet = new HashSet<>(expectedKeys); - assertEquals(expectedKeySet, actualKeySet); String getTelemetryValuesUrl; @@ -105,8 +91,8 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } else { getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); } - start = System.currentTimeMillis(); - end = System.currentTimeMillis() + 5000; + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; Map>> values = null; while (start <= end) { values = doGetAsyncTyped(getTelemetryValuesUrl, new TypeReference<>() {}); @@ -142,11 +128,7 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } else { assertValues(values, 0); } - } - - private void postTelemetry(CoapClient client, byte[] payload) throws IOException, ConnectorException { - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + client.disconnect(); } private void assertTs(Map>> deviceValues, List expectedKeys, int ts, int arrayIndex) { @@ -207,4 +189,20 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } } + private List getActualKeysList(DeviceId deviceId, List expectedKeys) throws Exception { + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; + + List actualKeys = null; + while (start <= end) { + actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {}); + if (actualKeys.size() == expectedKeys.size()) { + break; + } + Thread.sleep(100); + start += 100; + } + return actualKeys; + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java index 93e132663e..fb6d8aa0da 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; From 4ebab753c61eae65cb7509ac69b5f72766c93cd6 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 27 May 2022 12:03:24 +0300 Subject: [PATCH 17/80] refactoring coap-tests-speed-up --- .../server/transport/coap/CoapTestCallback.java | 15 +++++++++++++++ .../server/transport/coap/CoapTestClient.java | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java index 58379c9176..89fa13220d 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.transport.coap; import lombok.Data; diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java index 54e0a8cfe9..a6a3732ba2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.transport.coap; import org.eclipse.californium.core.CoapClient; From 8229ddfe7aeeeb2049cbd44c9ec97778d71cca75 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 27 May 2022 14:57:42 +0300 Subject: [PATCH 18/80] refactoring coap-tests-speed-up --- .../coap/AbstractCoapIntegrationTest.java | 7 +++-- ...AbstractCoapAttributesIntegrationTest.java | 31 ++++++++----------- .../CoapAttributesUpdatesIntegrationTest.java | 19 ++++++++++++ .../coap/claim/CoapClaimDeviceTest.java | 5 +-- .../coap/claim/CoapClaimProtoDeviceTest.java | 2 +- .../CoapProvisionJsonDeviceTest.java | 6 ++-- .../CoapProvisionProtoDeviceTest.java | 8 +---- ...tractCoapServerSideRpcIntegrationTest.java | 6 ++-- .../CoapAttributesIntegrationTest.java | 2 +- ...AbstractCoapTimeseriesIntegrationTest.java | 3 +- 10 files changed, 45 insertions(+), 44 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index 4f77f53a32..ce43ae3bb5 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -16,8 +16,6 @@ package org.thingsboard.server.transport.coap; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.junit.After; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.Device; @@ -41,7 +39,6 @@ import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadCon import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.AbstractTransportIntegrationTest; import static org.junit.Assert.assertEquals; @@ -54,8 +51,12 @@ import static org.junit.Assert.assertNotNull; public abstract class AbstractCoapIntegrationTest extends AbstractTransportIntegrationTest { protected final byte[] EMPTY_PAYLOAD = new byte[0]; + protected CoapTestClient client; protected void processAfterTest() throws Exception { + if (client != null) { + client.disconnect(); + } } protected void processBeforeTest(CoapTestConfigProperties config) throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index b6f5613791..9710c82866 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -15,36 +15,33 @@ */ package org.thingsboard.server.transport.coap.attributes; -import com.fasterxml.jackson.core.type.TypeReference; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.squareup.wire.schema.internal.parser.ProtoFileElement; -import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; import org.eclipse.californium.core.CoapObserveRelation; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.device.profile.*; -import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.common.transport.service.DefaultTransportService; -import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.transport.coap.CoapTestCallback; import org.thingsboard.server.transport.coap.CoapTestClient; -import org.thingsboard.server.transport.coap.attributes.updates.CoapAttributesUpdatesIntegrationTest; -import org.thingsboard.server.transport.mqtt.MqttTestCallback; -import org.thingsboard.server.transport.mqtt.MqttTestClient; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -54,7 +51,9 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.query.EntityKeyType.CLIENT_ATTRIBUTE; @@ -168,7 +167,7 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap } protected void processJsonTestRequestAttributesValuesFromTheServer() throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); SingleEntityFilter dtf = new SingleEntityFilter(); dtf.setSingleEntity(savedDevice.getId()); String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; @@ -195,11 +194,10 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; client.setURI(featureTokenUrl); validateJsonResponse(client.getMethod()); - client.disconnect(); } protected void processProtoTestRequestAttributesValuesFromTheServer() throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); SingleEntityFilter dtf = new SingleEntityFilter(); dtf.setSingleEntity(savedDevice.getId()); String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; @@ -226,7 +224,6 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; client.setURI(featureTokenUrl); validateProtoResponse(client.getMethod()); - client.disconnect(); } protected void processJsonTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { @@ -234,7 +231,7 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); } - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); CoapTestCallback callbackCoap = new CoapTestCallback(1); CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); @@ -262,7 +259,6 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap assertTrue(observeRelation.isCanceled()); awaitClientAfterCancelObserve(); - client.disconnect(); } protected void processProtoTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { @@ -270,7 +266,7 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); } - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); CoapTestCallback callbackCoap = new CoapTestCallback(1); CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); @@ -298,7 +294,6 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap assertTrue(observeRelation.isCanceled()); awaitClientAfterCancelObserve(); - client.disconnect(); } protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java index 343ee15afc..3214ae1639 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java @@ -16,19 +16,38 @@ package org.thingsboard.server.transport.coap.attributes.updates; import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.server.resources.Resource; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.coapserver.DefaultCoapServerService; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; +import org.thingsboard.server.transport.coap.CoapTransportResource; import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; +import static org.mockito.Mockito.spy; + @Slf4j @DaoSqlTest public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributesIntegrationTest { + CoapTransportResource coapTransportResource; + + @Autowired + DefaultCoapServerService defaultCoapServerService; + + @Autowired + DefaultTransportService defaultTransportService; + @Before public void beforeTest() throws Exception { + Resource api = defaultCoapServerService.getCoapServer().getRoot().getChild("api"); + coapTransportResource = spy( (CoapTransportResource) api.getChild("v1") ); + api.delete(api.getChild("v1") ); + api.add(coapTransportResource); CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() .deviceName("Test Subscribe to attribute updates") .build(); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java index 1e2c65ff35..e2be2280b2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java @@ -16,10 +16,8 @@ package org.thingsboard.server.transport.coap.claim; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; @@ -97,7 +95,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { log.warn("[testClaimingDevice] Device: {}, Transport type: {}", savedDevice.getName(), savedDevice.getType()); - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.CLAIM); + client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; byte[] failurePayloadBytes; if (emptyPayload) { @@ -144,7 +142,6 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { claimResponse = doPostClaimAsync("/api/customer/device/" + savedDevice.getName() + "/claim", claimRequest, ClaimResponse.class, status().isBadRequest()); assertEquals(claimResponse, ClaimResponse.CLAIMED); - client.disconnect(); } private void postClaimRequest(CoapTestClient client, byte[] payload) throws IOException, ConnectorException { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java index e4090cd07a..0096827af7 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java @@ -57,7 +57,7 @@ public class CoapClaimProtoDeviceTest extends CoapClaimDeviceTest { @Override protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.CLAIM); + client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; if (emptyPayload) { TransportApiProtos.ClaimDevice claimDevice = getClaimDevice(0, emptyPayload); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java index ded7b3a65b..ce5ca909ba 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java @@ -227,10 +227,8 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { private byte[] createCoapClientAndPublish(String deviceCredentials) throws Exception { String provisionRequestMsg = createTestProvisionMessage(deviceCredentials); - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.PROVISION); - byte[] responseByte = client.postMethod(provisionRequestMsg.getBytes()).getPayload(); - client.disconnect(); - return responseByte; + client = new CoapTestClient(accessToken, FeatureType.PROVISION); + return client.postMethod(provisionRequestMsg.getBytes()).getPayload(); } private String createTestProvisionMessage(String deviceCredentials) { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java index 74e39963a5..a5253b8409 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java @@ -16,10 +16,7 @@ package org.thingsboard.server.transport.coap.provision; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -47,8 +44,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509Ce import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; - @Slf4j @DaoSqlTest public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { @@ -231,10 +226,9 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { } private byte[] createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.PROVISION); + client = new CoapTestClient(accessToken, FeatureType.PROVISION); CoapResponse coapResponse = client.postMethod(provisionRequestMsg); Assert.assertNotNull("COAP response", coapResponse); - client.disconnect(); return coapResponse.getPayload(); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java index 36cd372b1f..beb4d3747c 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java @@ -71,7 +71,7 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC protected static final Long asyncContextTimeoutToUseRpcPlugin = 10000L; protected void processOneWayRpcTest(boolean protobuf) throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.RPC); + client = new CoapTestClient(accessToken, FeatureType.RPC); CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, true, protobuf); CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); @@ -87,11 +87,10 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); - client.disconnect(); } protected void processTwoWayRpcTest(String expectedResponseResult, boolean protobuf) throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.RPC); + client = new CoapTestClient(accessToken, FeatureType.RPC); CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, false, protobuf); CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); @@ -115,7 +114,6 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); - client.disconnect(); } protected void processOnLoadResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java index bd8c9a47f6..057e4366de 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java @@ -72,7 +72,7 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { protected void processAttributesTest(List expectedKeys, byte[] payload, boolean presenceFieldsTest) throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); CoapResponse coapResponse = client.postMethod(payload); assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java index 31c60f49f6..a1fba145a9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java @@ -73,7 +73,7 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } protected void processTestPostTelemetry(byte[] payloadBytes, List expectedKeys, boolean withTs, boolean presenceFieldsTest) throws Exception { - CoapTestClient client = new CoapTestClient(accessToken, FeatureType.TELEMETRY); + client = new CoapTestClient(accessToken, FeatureType.TELEMETRY); CoapResponse coapResponse = client.postMethod(payloadBytes); assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); @@ -128,7 +128,6 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } else { assertValues(values, 0); } - client.disconnect(); } private void assertTs(Map>> deviceValues, List expectedKeys, int ts, int arrayIndex) { From 6adf0f256e6cf183d36d203c66f96628f9ed1c4d Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 10 Jun 2022 14:48:20 +0300 Subject: [PATCH 19/80] Report device activity from edge service - set active flag to true on cloud --- .../service/edge/rpc/EdgeGrpcSession.java | 2 +- .../edge/rpc/processor/BaseEdgeProcessor.java | 10 ++++ .../rpc/processor/TelemetryEdgeProcessor.java | 54 +++++++++++++++++-- .../thingsboard/server/edge/BaseEdgeTest.java | 27 +++++++--- 4 files changed, 81 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 6c991444b2..bb67b760b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -571,7 +571,7 @@ public final class EdgeGrpcSession implements Closeable { try { if (uplinkMsg.getEntityDataCount() > 0) { for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) { - result.addAll(ctx.getTelemetryProcessor().processTelemetryFromEdge(edge.getTenantId(), edge.getCustomerId(), entityData)); + result.addAll(ctx.getTelemetryProcessor().processTelemetryFromEdge(edge.getTenantId(), edge.getCustomerId(), entityData, sessionId)); } } if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 09bef32af9..46b36a1932 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EdgeUtils; @@ -52,6 +53,8 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.edge.rpc.constructor.AdminSettingsMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AlarmMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AssetMsgConstructor; @@ -137,6 +140,13 @@ public abstract class BaseEdgeProcessor { @Autowired protected WidgetTypeService widgetTypeService; + @Autowired + protected PartitionService partitionService; + + @Autowired + @Lazy + protected TbQueueProducerProvider producerProvider; + @Autowired protected DataValidator deviceValidator; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java index 769d988507..cc17970ec8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java @@ -52,6 +52,8 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.util.JsonUtils; @@ -61,9 +63,12 @@ import org.thingsboard.server.gen.edge.v1.EntityDataProto; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.Nullable; +import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -77,14 +82,19 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { private final Gson gson = new Gson(); - public List> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData) { + private TbQueueProducer> tbCoreMsgProducer; + + @PostConstruct + public void init() { + tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); + } + + public List> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData, UUID edgeSessionId) { log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData); List> result = new ArrayList<>(); EntityId entityId = constructEntityId(entityData); if ((entityData.hasPostAttributesMsg() || entityData.hasPostTelemetryMsg() || entityData.hasAttributesUpdatedMsg()) && entityId != null) { - // @voba - in terms of performance we should not fetch device from DB by id - // TbMsgMetaData metaData = constructBaseMsgMetadata(tenantId, entityId); - TbMsgMetaData metaData = new TbMsgMetaData(); + TbMsgMetaData metaData = constructBaseMsgMetadata(tenantId, entityId); metaData.putValue(DataConstants.MSG_SOURCE_KEY, DataConstants.EDGE_MSG_SOURCE); if (entityData.hasPostAttributesMsg()) { result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData)); @@ -96,6 +106,11 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { if (entityData.hasPostTelemetryMsg()) { result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData)); } + if (EntityType.DEVICE.equals(entityId.getEntityType())) { + Device device = deviceService.findDeviceById(tenantId, new DeviceId(entityId.getId())); + // for edge context sessionId is exact edgeSessionId + reportActivity(device, edgeSessionId); + } } if (entityData.hasAttributeDeleteMsg()) { result.add(processAttributeDeleteMsg(tenantId, entityId, entityData.getAttributeDeleteMsg(), entityData.getEntityType())); @@ -103,6 +118,37 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { return result; } + private void reportActivity(Device device, UUID sessionId) { + TransportProtos.SessionInfoProto.Builder builder = TransportProtos.SessionInfoProto.newBuilder() + .setSessionIdMSB(sessionId.getMostSignificantBits()) + .setSessionIdLSB(sessionId.getLeastSignificantBits()) + .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(device.getTenantId().getId().getLeastSignificantBits()) + .setDeviceIdMSB(device.getId().getId().getMostSignificantBits()) + .setDeviceIdLSB(device.getId().getId().getLeastSignificantBits()) + .setDeviceName(device.getName()) + .setDeviceType(device.getType()) + .setDeviceProfileIdMSB(device.getDeviceProfileId().getId().getMostSignificantBits()) + .setDeviceProfileIdLSB(device.getDeviceProfileId().getId().getLeastSignificantBits()); + + if (device.getCustomerId() != null && !device.getCustomerId().isNullUid()) { + builder.setCustomerIdMSB(device.getCustomerId().getId().getMostSignificantBits()); + builder.setCustomerIdLSB(device.getCustomerId().getId().getLeastSignificantBits()); + } + reportActivity(device.getTenantId(), device.getId(), device.getUuidId(), builder.build()); + } + + private void reportActivity(TenantId tenantId, DeviceId deviceId, UUID routingKey, TransportProtos.SessionInfoProto sessionInfo) { + TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(false).setRpcSubscription(false) + .setLastActivityTime(System.currentTimeMillis()).build(); + TransportProtos.TransportToDeviceActorMsg msg = TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setSubscriptionInfo(subscriptionInfoProto).build(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(routingKey, + TransportProtos.ToCoreMsg.newBuilder().setToDeviceActorMsg(msg).build()), null); + } + private TbMsgMetaData constructBaseMsgMetadata(TenantId tenantId, EntityId entityId) { TbMsgMetaData metaData = new TbMsgMetaData(); switch (entityId.getEntityType()) { diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java index 61bb53b060..a2f72a86d2 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java @@ -1339,17 +1339,17 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { String timeseriesKey = "key"; String timeseriesValue = "25"; data.addProperty(timeseriesKey, timeseriesValue); - UplinkMsg.Builder uplinkMsgBuilder1 = UplinkMsg.newBuilder(); + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); EntityDataProto.Builder entityDataBuilder = EntityDataProto.newBuilder(); entityDataBuilder.setPostTelemetryMsg(JsonConverter.convertToTelemetryProto(data, System.currentTimeMillis())); entityDataBuilder.setEntityType(device.getId().getEntityType().name()); entityDataBuilder.setEntityIdMSB(device.getUuidId().getMostSignificantBits()); entityDataBuilder.setEntityIdLSB(device.getUuidId().getLeastSignificantBits()); testAutoGeneratedCodeByProtobuf(entityDataBuilder); - uplinkMsgBuilder1.addEntityData(entityDataBuilder.build()); + uplinkMsgBuilder.addEntityData(entityDataBuilder.build()); - testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder1); - edgeImitator.sendUplinkMsg(uplinkMsgBuilder1.build()); + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); JsonObject attributesData = new JsonObject(); String attributesKey = "test_attr"; @@ -1381,9 +1381,22 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { String attributeValuesUrl = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/attributes/" + DataConstants.SERVER_SCOPE; List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() {}); - Assert.assertEquals(2, attributes.size()); - var result = attributes.stream().filter(kv -> kv.get("key").equals(attributesKey)).filter(kv -> kv.get("value").equals(attributesValue)).findFirst(); - Assert.assertTrue(result.isPresent()); + + Assert.assertEquals(3, attributes.size()); + + Optional> activeAttributeOpt = getAttributeByKey("active", attributes); + Assert.assertTrue(activeAttributeOpt.isPresent()); + Map activeAttribute = activeAttributeOpt.get(); + Assert.assertEquals("true", activeAttribute.get("value")); + + Optional> customAttributeOpt = getAttributeByKey(attributesKey, attributes); + Assert.assertTrue(customAttributeOpt.isPresent()); + Map customAttribute = customAttributeOpt.get(); + Assert.assertEquals(attributesValue, customAttribute.get("value")); + } + + private Optional> getAttributeByKey(String key, List> attributes) { + return attributes.stream().filter(kv -> kv.get("key").equals(key)).findFirst(); } private Map>> loadDeviceTimeseries(Device device, String timeseriesKey) throws Exception { From 131cff88e47a7abc0833b04756c0a73a5dd03680 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 10 Jun 2022 14:52:47 +0300 Subject: [PATCH 20/80] BaseEdgeTest - Added DELETE attribute after test complete --- .../src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java index a2f72a86d2..924bec8282 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java @@ -1393,6 +1393,8 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { Assert.assertTrue(customAttributeOpt.isPresent()); Map customAttribute = customAttributeOpt.get(); Assert.assertEquals(attributesValue, customAttribute.get("value")); + + doDelete("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/SERVER_SCOPE?keys=" + attributesKey, String.class); } private Optional> getAttributeByKey(String key, List> attributes) { From c3a317b2439b438512027eec19aa1e06fa6164a7 Mon Sep 17 00:00:00 2001 From: devaskim Date: Thu, 16 Jun 2022 11:17:59 +0500 Subject: [PATCH 21/80] Export all RxJS API. --- .../home/models/widget-component.models.ts | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ui-ngx/src/app/modules/home/models/widget-component.models.ts b/ui-ngx/src/app/modules/home/models/widget-component.models.ts index 3bb727f4f5..98fc90c9ea 100644 --- a/ui-ngx/src/app/modules/home/models/widget-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/widget-component.models.ts @@ -50,7 +50,6 @@ import { WidgetTypeId } from '@shared/models/id/widget-type-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetLayout } from '@shared/models/dashboard.models'; import { formatValue, isDefined } from '@core/utils'; -import { forkJoin, of } from 'rxjs'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { @@ -78,7 +77,8 @@ import { PageLink } from '@shared/models/page/page-link'; import { SortOrder } from '@shared/models/page/sort-order'; import { DomSanitizer } from '@angular/platform-browser'; import { Router } from '@angular/router'; -import { catchError, map, mergeMap, switchMap } from 'rxjs/operators'; +import * as RxJS from 'rxjs'; +import * as RxJSOperators from 'rxjs/operators'; import { FormattedData } from '@home/components/widget/lib/maps/map-models'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { EntityId } from '@shared/models/id/entity-id'; @@ -202,7 +202,7 @@ export class WidgetContext { if (this.defaultSubscription) { return this.defaultSubscription.sendOneWayCommand(method, params, timeout, persistent, retries, additionalInfo, requestUUID); } else { - return of(null); + return RxJS.of(null); } }, sendTwoWayCommand: (method, params, timeout, persistent, @@ -210,14 +210,14 @@ export class WidgetContext { if (this.defaultSubscription) { return this.defaultSubscription.sendTwoWayCommand(method, params, timeout, persistent, retries, additionalInfo, requestUUID); } else { - return of(null); + return RxJS.of(null); } }, completedCommand: () => { if (this.defaultSubscription) { return this.defaultSubscription.completedCommand(); } else { - return of(null); + return RxJS.of(null); } } }; @@ -264,12 +264,9 @@ export class WidgetContext { private popoverComponents: TbPopoverComponent[] = []; rxjs = { - forkJoin, - of, - map, - mergeMap, - switchMap, - catchError + + ...RxJS, + ...RxJSOperators }; registerPopoverComponent(popoverComponent: TbPopoverComponent) { From a3f97f9c77554f4568be7d852a75aadf5e2fc782 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Thu, 16 Jun 2022 18:27:37 +0300 Subject: [PATCH 22/80] fix semicolon --- .../org/thingsboard/server/transport/coap/CoapTestClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java index a6a3732ba2..dc2a3f4c83 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java @@ -57,7 +57,7 @@ public class CoapTestClient { public void disconnect() { if (client != null) { client.shutdown(); - }; + } } public CoapResponse postMethod(String requestBody) throws ConnectorException, IOException { From 97571c6453664f37afd8d0acc706cc21731ebbcb Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 18 Jun 2022 10:57:34 +0300 Subject: [PATCH 23/80] kafka-client bumped up to 3.2.0, original org.apache.kafka.common.network.NetworkReceive with no changes, the custom NetworkReceive not modified --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 895127e0c3..45fc50ed0d 100755 --- a/pom.xml +++ b/pom.xml @@ -98,8 +98,9 @@ 5.0.2 0.2.1 - 2.8.0 + org.apache.kafka.common.network.NetworkReceive class in the application module. It addresses the issue https://issues.apache.org/jira/browse/KAFKA-4090. + Here is the source to track https://github.com/apache/kafka/tree/trunk/clients/src/main/java/org/apache/kafka/common/network --> + 3.2.0 4.1.1 2.57 2.7.7 From 1ee35c2ed46e98936cffbd4b770e20c5baa37524 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 13 Jun 2022 19:16:54 +0300 Subject: [PATCH 24/80] EntityViewService optimization using local cache and lifecycle event broadcasting --- .../server/actors/ActorSystemContext.java | 5 ++ .../DefaultTbEntityViewService.java | 46 ++++++++++++++++++- .../entityView/TbEntityViewService.java | 9 +++- .../queue/DefaultTbClusterService.java | 3 +- .../processing/AbstractConsumerService.java | 2 + .../DefaultTelemetrySubscriptionService.java | 12 ++--- .../dao/entityview/EntityViewService.java | 1 + .../plugin/ComponentLifecycleListener.java | 20 ++++++++ .../dao/entityview/EntityViewServiceImpl.java | 6 +++ 9 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 733cf75e64..c7312ce5c3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -80,6 +80,7 @@ import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; +import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; @@ -220,6 +221,10 @@ public class ActorSystemContext { @Getter private EntityViewService entityViewService; + @Autowired + @Getter + private TbEntityViewService tbEntityViewService; + @Autowired @Getter private TelemetrySubscriptionService tsSubService; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java index 2eee167e75..546a48d4a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java @@ -23,6 +23,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -40,8 +41,11 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; @@ -50,6 +54,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -57,12 +63,15 @@ import static org.apache.commons.lang3.StringUtils.isBlank; @Service @TbCoreComponent +@TbRuleEngineComponent @AllArgsConstructor @Slf4j public class DefaultTbEntityViewService extends AbstractTbEntityService implements TbEntityViewService { private final TimeseriesService tsService; + final Map>> localCache = new ConcurrentHashMap<>(); + @Override public EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException { ActionType actionType = entityView.getId() == null ? ActionType.ADDED : ActionType.UPDATED; @@ -71,6 +80,9 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen this.updateEntityViewAttributes(user, savedEntityView, existingEntityView); notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView, null, actionType, user); + localCache.computeIfAbsent(savedEntityView.getTenantId(), (k) -> new ConcurrentReferenceHashMap<>()).clear(); + tbClusterService.broadcastEntityStateChangeEvent(savedEntityView.getTenantId(), savedEntityView.getId(), + entityView.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); return savedEntityView; } catch (Exception e) { notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e); @@ -122,6 +134,9 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen entityViewService.deleteEntityView(tenantId, entityViewId); notificationEntityService.notifyDeleteEntity(tenantId, entityViewId, entityView, entityView.getCustomerId(), ActionType.DELETED, relatedEdgeIds, user, entityViewId.toString()); + + localCache.computeIfAbsent(tenantId, (k) -> new ConcurrentReferenceHashMap<>()).clear(); + tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityViewId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.ENTITY_VIEW), null, null, ActionType.DELETED, user, e, entityViewId.toString()); @@ -214,6 +229,35 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen } } + @Override + public ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId) { + Map> localCacheByTenant = localCache.computeIfAbsent(tenantId, (k) -> new ConcurrentReferenceHashMap<>()); + List fromLocalCache = localCacheByTenant.get(entityId); + if (fromLocalCache != null) { + return Futures.immediateFuture(fromLocalCache); + } + + ListenableFuture> future = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId); + + return Futures.transform(future, (entityViewList) -> { + localCacheByTenant.put(entityId, entityViewList); + return entityViewList; + }, MoreExecutors.directExecutor()); + } + + @Override + public void onComponentLifecycleMsg(ComponentLifecycleMsg componentLifecycleMsg) { + Map> localCacheByTenant = localCache.computeIfAbsent(componentLifecycleMsg.getTenantId(), (k) -> new ConcurrentReferenceHashMap<>()); + if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) { + localCacheByTenant.clear(); //we don't know which entity was mapped before deletion + } else { + EntityView entityView = entityViewService.findEntityViewById(componentLifecycleMsg.getTenantId(), new EntityViewId(componentLifecycleMsg.getEntityId().getId())); + if (entityView != null) { + localCacheByTenant.remove(entityView.getEntityId()); + } + } + } + private ListenableFuture> copyAttributesFromEntityToEntityView(EntityView entityView, String scope, Collection keys, SecurityUser user) throws ThingsboardException { EntityViewId entityId = entityView.getId(); if (keys != null && !keys.isEmpty()) { @@ -345,7 +389,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen @Override public void onFailure(Throwable t) { try { - logTimeseriesDeleted(entityView.getTenantId(),user, entityId, keys, t); + logTimeseriesDeleted(entityView.getTenantId(), user, entityId, keys, t); } catch (ThingsboardException e) { log.error("Failed to log timeseries delete", e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java index e7e150531b..f66113e5e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java @@ -15,16 +15,21 @@ */ package org.thingsboard.server.service.entitiy.entityView; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleListener; import org.thingsboard.server.service.security.model.SecurityUser; -public interface TbEntityViewService { +import java.util.List; + +public interface TbEntityViewService extends ComponentLifecycleListener { EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException; @@ -46,4 +51,6 @@ public interface TbEntityViewService { EntityView unassignEntityViewFromCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer, SecurityUser user) throws ThingsboardException; + + ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 6b475f5a5a..441da82127 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -364,13 +364,14 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { byte[] msgBytes = encodingService.encode(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); - Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); + Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) || entityType.equals(EntityType.DEVICE_PROFILE) || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) + || entityType.equals(EntityType.ENTITY_VIEW) || entityType.equals(EntityType.EDGE)) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 0c130c9f62..98fb95ad15 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -180,6 +180,8 @@ public abstract class AbstractConsumerService onTimeSeriesUpdate(tenantId, entityId, ts)); if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { - Futures.addCallback(this.entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), + Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), new FutureCallback>() { @Override public void onSuccess(@Nullable List result) { - if (result != null) { + if (result != null && !result.isEmpty()) { Map> tsMap = new HashMap<>(); for (TsKvEntry entry : ts) { tsMap.computeIfAbsent(entry.getKey(), s -> new ArrayList<>()).add(entry); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java index 36343c3662..884157fb7e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import java.util.List; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java new file mode 100644 index 0000000000..2bcba593e4 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.plugin; + +public interface ComponentLifecycleListener { + void onComponentLifecycleMsg(ComponentLifecycleMsg componentLifecycleMsg); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 6b1d6775d6..59763840ea 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -20,9 +20,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.event.TransactionalEventListener; +import org.springframework.util.ConcurrentReferenceHashMap; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -37,9 +41,11 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; From 380e9949514509984255d5d0565f111e95cc3f1b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 17 Jun 2022 19:11:25 +0300 Subject: [PATCH 25/80] TbEntityViewService bean lifecycle after removing TbCoreComponent annotation --- .../server/actors/ActorSystemContext.java | 3 ++- .../service/entitiy/AbstractTbEntityService.java | 16 ++++++++++------ .../entityView/DefaultTbEntityViewService.java | 4 ---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c7312ce5c3..28aaa583ab 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -221,7 +221,8 @@ public class ActorSystemContext { @Getter private EntityViewService entityViewService; - @Autowired + @Lazy + @Autowired(required = false) @Getter private TbEntityViewService tbEntityViewService; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 8439c1780b..7abe031f2e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -21,6 +21,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; @@ -93,13 +94,13 @@ public abstract class AbstractTbEntityService { @Autowired protected DbCallbackExecutorService dbExecutor; - @Autowired + @Autowired(required = false) protected TbNotificationEntityService notificationEntityService; @Autowired(required = false) protected EdgeService edgeService; @Autowired protected AlarmService alarmService; - @Autowired + @Autowired(required = false) protected EntityActionService entityActionService; @Autowired protected DeviceService deviceService; @@ -111,24 +112,27 @@ public abstract class AbstractTbEntityService { protected TenantService tenantService; @Autowired protected CustomerService customerService; - @Autowired + @Lazy + @Autowired(required = false) protected ClaimDevicesService claimDevicesService; @Autowired protected TbTenantProfileCache tenantProfileCache; @Autowired protected RuleChainService ruleChainService; - @Autowired + @Autowired(required = false) protected TbRuleChainService tbRuleChainService; - @Autowired + @Autowired(required = false) protected EdgeNotificationService edgeNotificationService; @Autowired protected QueueService queueService; @Autowired protected DashboardService dashboardService; + @Autowired protected EntitiesVersionControlService vcService; @Autowired protected EntityViewService entityViewService; + @Lazy @Autowired protected TelemetrySubscriptionService tsSubService; @Autowired @@ -149,7 +153,7 @@ public abstract class AbstractTbEntityService { protected InstallScripts installScripts; @Autowired protected UserService userService; - @Autowired + @Autowired(required = false) protected TbResourceService resourceService; @Autowired protected WidgetsBundleService widgetsBundleService; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java index 546a48d4a8..c7d447371f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java @@ -44,8 +44,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; @@ -62,8 +60,6 @@ import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.isBlank; @Service -@TbCoreComponent -@TbRuleEngineComponent @AllArgsConstructor @Slf4j public class DefaultTbEntityViewService extends AbstractTbEntityService implements TbEntityViewService { From 60348d91d58db2ce84cc061943d38788cf77f113 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 18 Jun 2022 10:54:30 +0300 Subject: [PATCH 26/80] DefaultTelemetrySubscriptionService - @Lazy TbEntityViewService --- .../service/telemetry/DefaultTelemetrySubscriptionService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 49fe3e696d..9ee4543b68 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.cluster.TbClusterService; @@ -83,7 +84,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public DefaultTelemetrySubscriptionService(AttributesService attrService, TimeseriesService tsService, - TbEntityViewService tbEntityViewService, + @Lazy TbEntityViewService tbEntityViewService, TbClusterService clusterService, PartitionService partitionService, TbApiUsageClient apiUsageClient, From 03f54839692cccfcec62603317d0b8f5485d7a51 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 18 Jun 2022 11:08:03 +0300 Subject: [PATCH 27/80] EntityViewServiceImpl cleanup imports --- .../server/dao/entityview/EntityViewServiceImpl.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 59763840ea..6b1d6775d6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -20,13 +20,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.event.TransactionalEventListener; -import org.springframework.util.ConcurrentReferenceHashMap; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -41,11 +37,9 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; From 718120b6141edd03152ae9fe5b0878808e424922 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 18 Jun 2022 11:12:04 +0300 Subject: [PATCH 28/80] EntitiViesService - cleanup imports --- .../thingsboard/server/dao/entityview/EntityViewService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java index 884157fb7e..bb790c78cb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java @@ -27,8 +27,6 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import java.util.List; From 59bcc2f600d162d0d5ff064bb3d309690e2ac52f Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Jun 2022 12:51:55 +0300 Subject: [PATCH 29/80] Code review changes --- .../edge/rpc/processor/BaseEdgeProcessor.java | 10 ---- .../rpc/processor/TelemetryEdgeProcessor.java | 47 +------------------ 2 files changed, 1 insertion(+), 56 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 46b36a1932..09bef32af9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EdgeUtils; @@ -53,8 +52,6 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.edge.rpc.constructor.AdminSettingsMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AlarmMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AssetMsgConstructor; @@ -140,13 +137,6 @@ public abstract class BaseEdgeProcessor { @Autowired protected WidgetTypeService widgetTypeService; - @Autowired - protected PartitionService partitionService; - - @Autowired - @Lazy - protected TbQueueProducerProvider producerProvider; - @Autowired protected DataValidator deviceValidator; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java index cc17970ec8..5858024292 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java @@ -52,8 +52,6 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.util.JsonUtils; @@ -63,12 +61,9 @@ import org.thingsboard.server.gen.edge.v1.EntityDataProto; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -82,13 +77,6 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { private final Gson gson = new Gson(); - private TbQueueProducer> tbCoreMsgProducer; - - @PostConstruct - public void init() { - tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); - } - public List> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData, UUID edgeSessionId) { log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData); List> result = new ArrayList<>(); @@ -107,9 +95,7 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData)); } if (EntityType.DEVICE.equals(entityId.getEntityType())) { - Device device = deviceService.findDeviceById(tenantId, new DeviceId(entityId.getId())); - // for edge context sessionId is exact edgeSessionId - reportActivity(device, edgeSessionId); + deviceStateService.onDeviceActivity(tenantId, new DeviceId(entityId.getId()), System.currentTimeMillis()); } } if (entityData.hasAttributeDeleteMsg()) { @@ -118,37 +104,6 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { return result; } - private void reportActivity(Device device, UUID sessionId) { - TransportProtos.SessionInfoProto.Builder builder = TransportProtos.SessionInfoProto.newBuilder() - .setSessionIdMSB(sessionId.getMostSignificantBits()) - .setSessionIdLSB(sessionId.getLeastSignificantBits()) - .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(device.getTenantId().getId().getLeastSignificantBits()) - .setDeviceIdMSB(device.getId().getId().getMostSignificantBits()) - .setDeviceIdLSB(device.getId().getId().getLeastSignificantBits()) - .setDeviceName(device.getName()) - .setDeviceType(device.getType()) - .setDeviceProfileIdMSB(device.getDeviceProfileId().getId().getMostSignificantBits()) - .setDeviceProfileIdLSB(device.getDeviceProfileId().getId().getLeastSignificantBits()); - - if (device.getCustomerId() != null && !device.getCustomerId().isNullUid()) { - builder.setCustomerIdMSB(device.getCustomerId().getId().getMostSignificantBits()); - builder.setCustomerIdLSB(device.getCustomerId().getId().getLeastSignificantBits()); - } - reportActivity(device.getTenantId(), device.getId(), device.getUuidId(), builder.build()); - } - - private void reportActivity(TenantId tenantId, DeviceId deviceId, UUID routingKey, TransportProtos.SessionInfoProto sessionInfo) { - TransportProtos.SubscriptionInfoProto subscriptionInfoProto = TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(false).setRpcSubscription(false) - .setLastActivityTime(System.currentTimeMillis()).build(); - TransportProtos.TransportToDeviceActorMsg msg = TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setSubscriptionInfo(subscriptionInfoProto).build(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); - tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(routingKey, - TransportProtos.ToCoreMsg.newBuilder().setToDeviceActorMsg(msg).build()), null); - } - private TbMsgMetaData constructBaseMsgMetadata(TenantId tenantId, EntityId entityId) { TbMsgMetaData metaData = new TbMsgMetaData(); switch (entityId.getEntityType()) { From 87834c2ccd33c4eefe1967412a48991ac1900ecf Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Jun 2022 12:52:49 +0300 Subject: [PATCH 30/80] Remove unused paramater --- .../thingsboard/server/service/edge/rpc/EdgeGrpcSession.java | 2 +- .../service/edge/rpc/processor/TelemetryEdgeProcessor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index cb00624f62..1d30d05fea 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -573,7 +573,7 @@ public final class EdgeGrpcSession implements Closeable { try { if (uplinkMsg.getEntityDataCount() > 0) { for (EntityDataProto entityData : uplinkMsg.getEntityDataList()) { - result.addAll(ctx.getTelemetryProcessor().processTelemetryFromEdge(edge.getTenantId(), edge.getCustomerId(), entityData, sessionId)); + result.addAll(ctx.getTelemetryProcessor().processTelemetryFromEdge(edge.getTenantId(), edge.getCustomerId(), entityData)); } } if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java index 5858024292..5c807f9434 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java @@ -77,7 +77,7 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { private final Gson gson = new Gson(); - public List> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData, UUID edgeSessionId) { + public List> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData) { log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData); List> result = new ArrayList<>(); EntityId entityId = constructEntityId(entityData); From 1d08e319bc4136d2c920aa9dfe3479f73ed1e1b4 Mon Sep 17 00:00:00 2001 From: devaskim Date: Wed, 22 Jun 2022 14:53:43 +0500 Subject: [PATCH 31/80] Add latest value to widget's legend. --- ui-ngx/src/app/core/api/widget-subscription.ts | 15 ++++++++++++++- .../widget/legend-config.component.html | 3 +++ .../components/widget/legend-config.component.ts | 6 ++++-- .../home/components/widget/legend.component.html | 8 ++++++++ .../home/components/widget/legend.component.ts | 3 ++- ui-ngx/src/app/shared/models/widget.models.ts | 5 ++++- .../src/assets/locale/locale.constant-en_US.json | 2 ++ 7 files changed, 37 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/core/api/widget-subscription.ts b/ui-ngx/src/app/core/api/widget-subscription.ts index 8cbcd19918..63dbe27739 100644 --- a/ui-ngx/src/app/core/api/widget-subscription.ts +++ b/ui-ngx/src/app/core/api/widget-subscription.ts @@ -269,7 +269,8 @@ export class WidgetSubscription implements IWidgetSubscription { (this.legendConfig.showMin === true || this.legendConfig.showMax === true || this.legendConfig.showAvg === true || - this.legendConfig.showTotal === true); + this.legendConfig.showTotal === true || + this.legendConfig.showLatest === true); this.initDataSubscription().subscribe(() => { subscriptionSubject.next(this); subscriptionSubject.complete(); @@ -1283,6 +1284,7 @@ export class WidgetSubscription implements IWidgetSubscription { max: null, avg: null, total: null, + latest: null, hidden: false }; this.legendData.data.push(legendKeyData); @@ -1439,6 +1441,9 @@ export class WidgetSubscription implements IWidgetSubscription { if (this.legendConfig.showTotal) { legendKeyData.total = this.ctx.widgetUtils.formatValue(calculateTotal(data), decimals, units); } + if (this.legendConfig.showLatest) { + legendKeyData.latest = this.ctx.widgetUtils.formatValue(calculateLatest(data), decimals, units); + } this.callbacks.legendDataUpdated(this, detectChanges !== false); } @@ -1511,3 +1516,11 @@ function calculateTotal(data: DataSet): number { return null; } } + +function calculateLatest(data: DataSet): number { + if (data.length > 0) { + return Number(data[data.length - 1][1]); + } else { + return null; + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html index bf7a27d4af..6e08046a22 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html @@ -50,6 +50,9 @@ {{ 'legend.show-total' | translate }} + + {{ 'legend.show-latest' | translate }} + {{ 'legend.sort-legend' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts index 588e1237e7..16e32bc866 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts @@ -66,7 +66,8 @@ export class LegendConfigComponent implements OnInit, OnDestroy, ControlValueAcc showMin: [null, []], showMax: [null, []], showAvg: [null, []], - showTotal: [null, []] + showTotal: [null, []], + showLatest: [null, []] }); this.legendSettingsFormDirectionChanges$ = this.legendConfigForm.get('direction').valueChanges .subscribe((direction: LegendDirection) => { @@ -124,7 +125,8 @@ export class LegendConfigComponent implements OnInit, OnDestroy, ControlValueAcc showMin: isDefined(legendConfig.showMin) ? legendConfig.showMin : false, showMax: isDefined(legendConfig.showMax) ? legendConfig.showMax : false, showAvg: isDefined(legendConfig.showAvg) ? legendConfig.showAvg : false, - showTotal: isDefined(legendConfig.showTotal) ? legendConfig.showTotal : false + showTotal: isDefined(legendConfig.showTotal) ? legendConfig.showTotal : false, + showLatest: isDefined(legendConfig.showLatest) ? legendConfig.showLatest : false }, {emitEvent: false}); } this.onDirectionChanged(legendConfig.direction); diff --git a/ui-ngx/src/app/modules/home/components/widget/legend.component.html b/ui-ngx/src/app/modules/home/components/widget/legend.component.html index bbead89d6b..5b30af3aac 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/legend.component.html @@ -24,6 +24,7 @@ {{ 'legend.max' | translate }} {{ 'legend.avg' | translate }} {{ 'legend.total' | translate }} + {{ 'legend.latest' | translate }} @@ -39,6 +40,7 @@ {{ legendData.data[legendKey.dataIndex].max }} {{ legendData.data[legendKey.dataIndex].avg }} {{ legendData.data[legendKey.dataIndex].total }} + {{ legendData.data[legendKey.dataIndex].latest }} @@ -76,5 +78,11 @@ {{ legendData.data[legendKey.dataIndex].total }} + + {{ 'legend.latest' | translate }} + + {{ legendData.data[legendKey.dataIndex].latest }} + + diff --git a/ui-ngx/src/app/modules/home/components/widget/legend.component.ts b/ui-ngx/src/app/modules/home/components/widget/legend.component.ts index 0074280732..4548621517 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/legend.component.ts @@ -43,7 +43,8 @@ export class LegendComponent implements OnInit { this.displayHeader = this.legendConfig.showMin === true || this.legendConfig.showMax === true || this.legendConfig.showAvg === true || - this.legendConfig.showTotal === true; + this.legendConfig.showTotal === true || + this.legendConfig.showLatest === true; this.isHorizontal = this.legendConfig.position === LegendPosition.bottom || this.legendConfig.position === LegendPosition.top; diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index 703cffcaee..9450efbf9c 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -227,6 +227,7 @@ export interface LegendConfig { showMax: boolean; showAvg: boolean; showTotal: boolean; + showLatest: boolean; } export function defaultLegendConfig(wType: widgetType): LegendConfig { @@ -237,7 +238,8 @@ export function defaultLegendConfig(wType: widgetType): LegendConfig { showMin: false, showMax: false, showAvg: wType === widgetType.timeseries, - showTotal: false + showTotal: false, + showLatest: false }; } @@ -323,6 +325,7 @@ export interface LegendKeyData { max: string; avg: string; total: string; + latest: string; hidden: boolean; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index aab433c992..e0e11702ef 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2341,11 +2341,13 @@ "show-min": "Show min value", "show-avg": "Show average value", "show-total": "Show total value", + "show-latest": "Show latest value", "settings": "Legend settings", "min": "min", "max": "max", "avg": "avg", "total": "total", + "latest": "latest", "comparison-time-ago": { "previousInterval": "(previous interval)", "customInterval": "(custom interval)", From e551c53b49eeb0248c2cf489728d4f2d53c161b4 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 22 Jun 2022 14:38:02 +0300 Subject: [PATCH 32/80] Swagger docs for EntitiesVersionControlController --- .../controller/ControllerConstants.java | 1 + .../EntitiesVersionControlController.java | 316 +++++++++++++----- 2 files changed, 241 insertions(+), 76 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index b83d029d95..3e4e2e6e87 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -137,6 +137,7 @@ public class ControllerConstants { protected static final String EDGE_ASSIGN_RECEIVE_STEP_DESCRIPTION = "(Edge will receive this instantly, if it's currently connected, or once it's going to be connected to platform). "; protected static final String ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the entity version name."; + protected static final String VERSION_ID_PARAM_DESCRIPTION = "Version id, for example fd82625bdd7d6131cf8027b44ee967012ecaf990. Represents commit hash."; protected static final String MARKDOWN_CODE_BLOCK_START = "```json\n"; protected static final String MARKDOWN_CODE_BLOCK_END = "\n```"; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 1bcc304548..11d64b0be4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -57,14 +57,19 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.VC_REQUEST_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.VERSION_ID_PARAM_DESCRIPTION; @RestController @TbCoreComponent @@ -76,9 +81,20 @@ public class EntitiesVersionControlController extends BaseController { private final EntitiesVersionControlService versionControlService; - @ApiOperation(value = "", notes = "" + - "SINGLE_ENTITY:" + NEW_LINE + - "```\n{\n" + + @ApiOperation(value = "Save entities version (saveEntitiesVersion)", notes = "" + + "Creates a new version of entities (or a single entity) by request.\n" + + "Supported entity types: CUSTOMER, ASSET, RULE_CHAIN, DASHBOARD, DEVICE_PROFILE, DEVICE, ENTITY_VIEW, WIDGETS_BUNDLE." + NEW_LINE + + "There are two available types of request: `SINGLE_ENTITY` and `COMPLEX`. " + + "Each of them contains version name (`versionName`) and name of a branch (`branch`) to create version (commit) in. " + + "If specified branch does not exists in a remote repo, then new empty branch will be created. " + + "Request of the `SINGLE_ENTITY` type has id of an entity (`entityId`) and additional configuration (`config`) " + + "which has following options: \n" + + "- `saveRelations` - whether to add inbound and outbound relations of type COMMON to created entity version;\n" + + "- `saveAttributes` - to save attributes of server scope (and also shared scope for devices);\n" + + "- `saveCredentials` - when saving a version of a device, to add its credentials to the version." + NEW_LINE + + "An example of a `SINGLE_ENTITY` version create request:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + " \"type\": \"SINGLE_ENTITY\",\n" + "\n" + " \"versionName\": \"Version 1.0\",\n" + @@ -89,11 +105,25 @@ public class EntitiesVersionControlController extends BaseController { " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + " },\n" + " \"config\": {\n" + - " \"saveRelations\": true\n" + + " \"saveRelations\": true,\n" + + " \"saveAttributes\": true,\n" + + " \"saveCredentials\": false\n" + " }\n" + - "}\n```" + NEW_LINE + - "COMPLEX:" + NEW_LINE + - "```\n{\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "Second request type (`COMPLEX`), additionally to `branch` and `versionName`, contains following properties:\n" + + "- `entityTypes` - a structure with entity types to export and configuration for each entity type; " + + " this configuration has all the options available for `SINGLE_ENTITY` and additionally has these ones: \n" + + " - `allEntities` and `entityIds` - if you want to save the version of all entities of the entity type " + + " then set `allEntities` param to true, otherwise set it to false and specify the list of specific entities (`entityIds`);\n" + + " - `syncStrategy` - synchronization strategy to use for this entity type: when set to `OVERWRITE` " + + " then the list of remote entities of this type will be overwritten by newly added entities. If set to " + + " `MERGE` - existing remote entities of this entity type will not be removed, new entities will just " + + " be added on top (or existing remote entities will be updated).\n" + + "- `syncStrategy` - default synchronization strategy to use when it is not specified for an entity type." + NEW_LINE + + "Example for this type of request:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + " \"type\": \"COMPLEX\",\n" + "\n" + " \"versionName\": \"Devices and profiles: release 2\",\n" + @@ -104,7 +134,9 @@ public class EntitiesVersionControlController extends BaseController { " \"DEVICE\": {\n" + " \"syncStrategy\": null,\n" + " \"allEntities\": true,\n" + - " \"saveRelations\": true\n" + + " \"saveRelations\": true,\n" + + " \"saveAttributes\": true,\n" + + " \"saveCredentials\": true\n" + " },\n" + " \"DEVICE_PROFILE\": {\n" + " \"syncStrategy\": \"MERGE\",\n" + @@ -115,7 +147,11 @@ public class EntitiesVersionControlController extends BaseController { " \"saveRelations\": true\n" + " }\n" + " }\n" + - "}\n```") + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "Response wil contain generated request UUID, that can be then used to retrieve " + + "status of operation via `getVersionCreateRequestStatus`.\n" + + TENANT_AUTHORITY_PARAGRAPH) @PostMapping("/version") public DeferredResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws Exception { SecurityUser user = getCurrentUser(); @@ -123,7 +159,32 @@ public class EntitiesVersionControlController extends BaseController { return wrapFuture(versionControlService.saveEntitiesVersion(user, request)); } - @ApiOperation(value = "", notes = "") + @ApiOperation(value = "Get version create request status (getVersionCreateRequestStatus)", notes = "" + + "Returns the status of previously made version create request. " + NEW_LINE + + "This status contains following properties:\n" + + "- `done` - whether request processing is finished;\n" + + "- `version` - created version info: timestamp, version id (commit hash), commit name and commit author;\n" + + "- `added` - count of items that were created in the remote repo;\n" + + "- `modified` - modified items count;\n" + + "- `removed` - removed items count;\n" + + "- `error` - error message, if an error occurred while handling the request." + NEW_LINE + + "An example of successful status:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"done\": true,\n" + + " \"added\": 10,\n" + + " \"modified\": 2,\n" + + " \"removed\": 5,\n" + + " \"version\": {\n" + + " \"timestamp\": 1655198528000,\n" + + " \"id\":\"8a834dd389ed80e0759ba8ee338b3f1fd160a114\",\n" + + " \"name\": \"My devices v2.0\",\n" + + " \"author\": \"John Doe\"\n" + + " },\n" + + " \"error\": null\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{requestId}/status") public VersionCreationResult getVersionCreateRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true) @PathVariable UUID requestId) throws Exception { @@ -131,13 +192,42 @@ public class EntitiesVersionControlController extends BaseController { return versionControlService.getVersionCreateStatus(getCurrentUser(), requestId); } - @ApiOperation(value = "", notes = "" + - "```\n[\n" + - " {\n" + - " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + - " \"name\": \"Device profile 1 version 1.0\"\n" + - " }\n" + - "]\n```") + @ApiOperation(value = "List entity versions (listEntityVersions)", notes = "" + + "Returns list of versions for a specific entity in a concrete branch. \n" + + "You need to specify external id of an entity to list versions for. This is `externalId` property of an entity, " + + "or otherwise if not set - simply id of this entity. \n" + + "If specified branch does not exist - empty page data will be returned. " + NEW_LINE + + "Each version info item has timestamp, id, name and author. Version id can then be used to restore the version. " + + PAGE_DATA_PARAMETERS + NEW_LINE + + "Response example: \n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"data\": [\n" + + " {\n" + + " \"timestamp\": 1655198593000,\n" + + " \"id\": \"fd82625bdd7d6131cf8027b44ee967012ecaf990\",\n" + + " \"name\": \"Devices and assets - v2.0\",\n" + + " \"author\": \"John Doe \"\n" + + " },\n" + + " {\n" + + " \"timestamp\": 1655198528000,\n" + + " \"id\": \"682adcffa9c8a2f863af6f00c4850323acbd4219\",\n" + + " \"name\": \"Update my device\",\n" + + " \"author\": \"John Doe \"\n" + + " },\n" + + " {\n" + + " \"timestamp\": 1655198280000,\n" + + " \"id\": \"d2a6087c2b30e18cc55e7cdda345a8d0dfb959a4\",\n" + + " \"name\": \"Devices and assets - v1.0\",\n" + + " \"author\": \"John Doe \"\n" + + " }\n" + + " ],\n" + + " \"totalPages\": 1,\n" + + " \"totalElements\": 3,\n" + + " \"hasNext\": false\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) public DeferredResult> listEntityVersions(@PathVariable String branch, @PathVariable EntityType entityType, @@ -158,13 +248,12 @@ public class EntitiesVersionControlController extends BaseController { return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink)); } - @ApiOperation(value = "", notes = "" + - "```\n[\n" + - " {\n" + - " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + - " \"name\": \"Device profiles from dev\"\n" + - " }\n" + - "]\n```") + @ApiOperation(value = "List entity type versions (listEntityTypeVersions)", notes = "" + + "Returns list of versions of an entity type in a branch. This is a collected list of versions that were created " + + "for entities of this type in a remote branch. \n" + + "If specified branch does not exist - empty page data will be returned. " + + "The response structure is the same as for `listEntityVersions` API method." + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) public DeferredResult> listEntityTypeVersions(@PathVariable String branch, @PathVariable EntityType entityType, @@ -183,21 +272,11 @@ public class EntitiesVersionControlController extends BaseController { return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink)); } - @ApiOperation(value = "", notes = "" + - "```\n[\n" + - " {\n" + - " \"id\": \"ba9baaca1742b730e7331f31a6a51da5fc7da7f7\",\n" + - " \"name\": \"Device 1 removed\"\n" + - " },\n" + - " {\n" + - " \"id\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + - " \"name\": \"Device profiles added\"\n" + - " },\n" + - " {\n" + - " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + - " \"name\": \"Devices added\"\n" + - " }\n" + - "]\n```") + @ApiOperation(value = "List all versions (listVersions)", notes = "" + + "Lists all available versions in a branch for all entity types. \n" + + "If specified branch does not exist - empty page data will be returned. " + + "The response format is the same as for `listEntityVersions` API method." + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) public DeferredResult> listVersions(@PathVariable String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @@ -216,23 +295,42 @@ public class EntitiesVersionControlController extends BaseController { } + @ApiOperation(value = "List entities at version (listEntitiesAtVersion)", notes = "" + + "Returns a list of remote entities of a specific entity type that are available at a concrete version. \n" + + "Each entity item in the result has `externalId` property. " + + "Entities order will be the same as in the repository." + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping("/entity/{branch}/{entityType}/{versionId}") public DeferredResult> listEntitiesAtVersion(@PathVariable String branch, @PathVariable EntityType entityType, + @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); } + @ApiOperation(value = "List all entities at version (listAllEntitiesAtVersion)", notes = "" + + "Returns a list of all remote entities available in a specific version. " + + "Response type is the same as for listAllEntitiesAtVersion API method. \n" + + "Returned entities order will be the same as in the repository." + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping("/entity/{branch}/{versionId}") public DeferredResult> listAllEntitiesAtVersion(@PathVariable String branch, + @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); } + @ApiOperation(value = "Get entity data info (getEntityDataInfo)", notes = "" + + "Retrieves short info about the remote entity by external id at a concrete version. \n" + + "Returned entity data info contains following properties: " + + "`hasRelations` (whether stored entity data contains relations), `hasAttributes` (contains attributes) and " + + "`hasCredentials` (whether stored device data has credentials)." + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping("/info/{versionId}/{entityType}/{externalEntityUuid}") - public DeferredResult getEntityDataInfo(@PathVariable String versionId, + public DeferredResult getEntityDataInfo(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) + @PathVariable String versionId, @PathVariable EntityType entityType, @PathVariable UUID externalEntityUuid) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); @@ -240,19 +338,33 @@ public class EntitiesVersionControlController extends BaseController { return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); } + @ApiOperation(value = "Compare entity data to version (compareEntityDataToVersion)", notes = "" + + "Returns an object with current entity data and the one at a specific version. " + + "Entity data structure is the same as stored in a repository. " + + TENANT_AUTHORITY_PARAGRAPH) @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") public DeferredResult compareEntityDataToVersion(@PathVariable String branch, @PathVariable EntityType entityType, @PathVariable UUID internalEntityUuid, + @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @RequestParam String versionId) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId)); } - @ApiOperation(value = "", notes = "" + - "SINGLE_ENTITY:" + NEW_LINE + - "```\n{\n" + + @ApiOperation(value = "Load entities version (loadEntitiesVersion)", notes = "" + + "Loads specific version of remote entities (or single entity) by request. " + + "Supported entity types: CUSTOMER, ASSET, RULE_CHAIN, DASHBOARD, DEVICE_PROFILE, DEVICE, ENTITY_VIEW, WIDGETS_BUNDLE." + NEW_LINE + + "There are multiple types of request. Each of them requires branch name (`branch`) and version id (`versionId`). " + + "Request of type `SINGLE_ENTITY` is needed to restore a concrete version of a specific entity. It contains " + + "id of a remote entity (`externalEntityId`) and additional configuration (`config`):\n" + + "- `loadRelations` - to update relations list (in case `saveRelations` option was enabled during version creation);\n" + + "- `loadAttributes` - to load entity attributes (if `saveAttributes` config option was enabled);\n" + + "- `loadCredentials` - to update device credentials (if `saveCredentials` option was enabled during version creation)." + NEW_LINE + + "An example of such request:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + " \"type\": \"SINGLE_ENTITY\",\n" + " \n" + " \"branch\": \"dev\",\n" + @@ -264,11 +376,23 @@ public class EntitiesVersionControlController extends BaseController { " },\n" + " \"config\": {\n" + " \"loadRelations\": false,\n" + - " \"findExistingEntityByName\": false\n" + + " \"loadAttributes\": true,\n" + + " \"loadCredentials\": true\n" + " }\n" + - "}\n```" + NEW_LINE + - "ENTITY_TYPE:" + NEW_LINE + - "```\n{\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "Another request type (`ENTITY_TYPE`) is needed to load specific version of the whole entity types. " + + "It contains a structure with entity types to load and configs for each entity type (`entityTypes`). " + + "For each specified entity type, the method will load all remote entities of this type that are present " + + "at the version. A config for each entity type contains the same options as in `SINGLE_ENTITY` request type, and " + + "additionally contains following options:\n" + + "- `removeOtherEntities` - to remove local entities that are not present on the remote - basically to " + + " overwrite local entity type with the remote one;\n" + + "- `findExistingEntityByName` - when you are loading some remote entities that are not yet present at this tenant, " + + " try to find existing entity by name and update it rather than create new." + NEW_LINE + + "Here is an example of the request to completely restore version of the whole device entity type:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + " \"type\": \"ENTITY_TYPE\",\n" + "\n" + " \"branch\": \"dev\",\n" + @@ -276,12 +400,18 @@ public class EntitiesVersionControlController extends BaseController { "\n" + " \"entityTypes\": {\n" + " \"DEVICE\": {\n" + - " \"loadRelations\": false,\n" + + " \"removeOtherEntities\": true,\n" + " \"findExistingEntityByName\": false,\n" + - " \"removeOtherEntities\": true\n" + + " \"loadRelations\": true,\n" + + " \"loadAttributes\": true,\n" + + " \"loadCredentials\": true\n" + " }\n" + " }\n" + - "}\n```") + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "The response will contain generated request UUID that is to be used to check the status of operation " + + "via `getVersionLoadRequestStatus`." + + TENANT_AUTHORITY_PARAGRAPH) @PostMapping("/entity") public UUID loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws Exception { SecurityUser user = getCurrentUser(); @@ -289,17 +419,54 @@ public class EntitiesVersionControlController extends BaseController { return versionControlService.loadEntitiesVersion(user, request); } - @ApiOperation(value = "", notes = "") + @ApiOperation(value = "Get version load request status (getVersionLoadRequestStatus)", notes = "" + + "Returns the status of previously made version load request. " + + "The structure contains following parameters:\n" + + "- `done` - if the request was successfully processed;\n" + + "- `result` - a list of load results for each entity type:\n" + + " - `created` - created entities count;\n" + + " - `updated` - updated entities count;\n" + + " - `deleted` - removed entities count.\n" + + "- `error` - if an error occurred during processing, error info:\n" + + " - `type` - error type;\n" + + " - `source` - an external id of remote entity;\n" + + " - `target` - if failed to find referenced entity by external id - this external id;\n" + + " - `message` - error message." + NEW_LINE + + "An example of successfully processed request status:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"done\": true,\n" + + " \"result\": [\n" + + " {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"created\": 10,\n" + + " \"updated\": 5,\n" + + " \"deleted\": 5\n" + + " },\n" + + " {\n" + + " \"entityType\": \"ASSET\",\n" + + " \"created\": 4,\n" + + " \"updated\": 0,\n" + + " \"deleted\": 8\n" + + " }\n" + + " ]\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + TENANT_AUTHORITY_PARAGRAPH + ) @GetMapping(value = "/entity/{requestId}/status") public VersionLoadResult getVersionLoadRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true) - @PathVariable UUID requestId) throws Exception { + @PathVariable UUID requestId) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return versionControlService.getVersionLoadStatus(getCurrentUser(), requestId); } - @ApiOperation(value = "", notes = "" + - "```\n[\n" + + @ApiOperation(value = "List branches (listBranches)", notes = "" + + "Lists branches available in the remote repository. \n\n" + + "Response example: \n" + + MARKDOWN_CODE_BLOCK_START + + "[\n" + " {\n" + " \"name\": \"master\",\n" + " \"default\": true\n" + @@ -312,31 +479,28 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"dev-2\",\n" + " \"default\": false\n" + " }\n" + - "]\n\n```") + "]" + + MARKDOWN_CODE_BLOCK_END) @GetMapping("/branches") - public DeferredResult> listBranches() throws ThingsboardException { - try { - accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); - final TenantId tenantId = getTenantId(); - ListenableFuture> branches = versionControlService.listBranches(tenantId); - return wrapFuture(Futures.transform(branches, remoteBranches -> { - List infos = new ArrayList<>(); + public DeferredResult> listBranches() throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + final TenantId tenantId = getTenantId(); + ListenableFuture> branches = versionControlService.listBranches(tenantId); + return wrapFuture(Futures.transform(branches, remoteBranches -> { + List infos = new ArrayList<>(); - String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); - if (StringUtils.isNotEmpty(defaultBranch)) { - infos.add(new BranchInfo(defaultBranch, true)); - } + String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + infos.add(new BranchInfo(defaultBranch, true)); + } - remoteBranches.forEach(branch -> { - if (!branch.equals(defaultBranch)) { - infos.add(new BranchInfo(branch, false)); - } - }); - return infos; - }, MoreExecutors.directExecutor())); - } catch (Exception e) { - throw handleException(e); - } + remoteBranches.forEach(branch -> { + if (!branch.equals(defaultBranch)) { + infos.add(new BranchInfo(branch, false)); + } + }); + return infos; + }, MoreExecutors.directExecutor())); } @Data From a3944c50477c69748022922f3663ce6acbe808a4 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Jun 2022 15:56:50 +0300 Subject: [PATCH 33/80] Introduced new proto for device activity --- .../device/DeviceActorMessageProcessor.java | 7 +++++ .../edge/rpc/processor/BaseEdgeProcessor.java | 10 +++++++ .../rpc/processor/TelemetryEdgeProcessor.java | 30 ++++++++++++++++++- common/cluster-api/src/main/proto/queue.proto | 5 ++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index b0830317a7..28c7a6a499 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -418,6 +418,9 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasUplinkNotificationMsg()) { processUplinkNotificationMsg(context, sessionInfo, msg.getUplinkNotificationMsg()); } + if (msg.hasDeviceActivity()) { + handleDeviceActivity(context, msg.getDeviceActivity()); + } callback.onSuccess(); } @@ -729,6 +732,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } + private void handleDeviceActivity(TbActorCtx context, TransportProtos.DeviceActivityProto deviceActivityProto) { + systemContext.getDeviceStateService().onDeviceActivity(tenantId, deviceId, deviceActivityProto.getLastActivityTime()); + } + void processCredentialsUpdate(TbActorMsg msg) { if (((DeviceCredentialsUpdateNotificationMsg) msg).getDeviceCredentials().getCredentialsType() == DeviceCredentialsType.LWM2M_CREDENTIALS) { sessions.forEach((k, v) -> { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index c67f7a5a14..8f37d743b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EdgeUtils; @@ -53,6 +54,8 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.edge.rpc.constructor.AdminSettingsMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AlarmMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AssetMsgConstructor; @@ -142,6 +145,13 @@ public abstract class BaseEdgeProcessor { @Autowired protected OtaPackageService otaPackageService; + @Autowired + protected PartitionService partitionService; + + @Autowired + @Lazy + protected TbQueueProducerProvider producerProvider; + @Autowired protected DataValidator deviceValidator; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java index 5c807f9434..88ea417c24 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java @@ -52,6 +52,8 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.util.JsonUtils; @@ -61,9 +63,12 @@ import org.thingsboard.server.gen.edge.v1.EntityDataProto; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.Nullable; +import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -77,6 +82,13 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { private final Gson gson = new Gson(); + private TbQueueProducer> tbCoreMsgProducer; + + @PostConstruct + public void init() { + tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); + } + public List> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData) { log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData); List> result = new ArrayList<>(); @@ -95,7 +107,23 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData)); } if (EntityType.DEVICE.equals(entityId.getEntityType())) { - deviceStateService.onDeviceActivity(tenantId, new DeviceId(entityId.getId()), System.currentTimeMillis()); + DeviceId deviceId = new DeviceId(entityId.getId()); + + TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()).build(); + + TransportProtos.DeviceActivityProto deviceActivityProto = TransportProtos.DeviceActivityProto.newBuilder() + .setLastActivityTime(System.currentTimeMillis()).build(); + + TransportProtos.TransportToDeviceActorMsg msg = TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) + .setDeviceActivity(deviceActivityProto).build(); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), + TransportProtos.ToCoreMsg.newBuilder().setToDeviceActorMsg(msg).build()), null); } } if (entityData.hasAttributeDeleteMsg()) { diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 3079978bce..7fefe18d4a 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -454,6 +454,10 @@ message GetOtaPackageResponseMsg { string fileName = 8; } +message DeviceActivityProto { + int64 lastActivityTime = 1; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -483,6 +487,7 @@ message TransportToDeviceActorMsg { ToDeviceRpcResponseStatusMsg rpcResponseStatusMsg = 10; SendPendingRPCMsg sendPendingRPC = 11; UplinkNotificationMsg uplinkNotificationMsg = 12; + DeviceActivityProto deviceActivity = 13; } message TransportToRuleEngineMsg { From 2aee04812e37771f950d2bbdd2563025bba58013 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Jun 2022 16:36:13 +0300 Subject: [PATCH 34/80] Bump Kafka broker version to 3.2.0 --- docker/docker-compose.kafka.yml | 2 +- docker/kafka.env | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 09c4554562..1683114605 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -19,7 +19,7 @@ version: '2.2' services: kafka: restart: always - image: "wurstmeister/kafka:2.13-2.6.0" + image: "bitnami/kafka:3.2.0" ports: - "9092:9092" env_file: diff --git a/docker/kafka.env b/docker/kafka.env index bd23f99ad1..9c28885252 100644 --- a/docker/kafka.env +++ b/docker/kafka.env @@ -1,12 +1,11 @@ - -KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 -KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 -KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 -KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT -KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE -KAFKA_CREATE_TOPICS=js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 -KAFKA_AUTO_CREATE_TOPICS_ENABLE=false -KAFKA_LOG_RETENTION_BYTES=1073741824 -KAFKA_LOG_SEGMENT_BYTES=268435456 -KAFKA_LOG_RETENTION_MS=300000 -KAFKA_LOG_CLEANUP_POLICY=delete +ALLOW_PLAINTEXT_LISTENER=yes +KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 +KAFKA_CFG_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 +KAFKA_CFG_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 +KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT +KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=false +KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INSIDE +KAFKA_CFG_LOG_RETENTION_BYTES=1073741824 +KAFKA_CFG_SEGMENT_BYTES=268435456 +KAFKA_CFG_LOG_RETENTION_MS=300000 +KAFKA_CFG_LOG_CLEANUP_POLICY=delete From 5e65438ebb4aa0ba1eabbf07d842bcc83f5f3656 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Jun 2022 16:51:40 +0300 Subject: [PATCH 35/80] Add EdgeService to the WidgetContext --- ui-ngx/src/app/modules/home/models/widget-component.models.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui-ngx/src/app/modules/home/models/widget-component.models.ts b/ui-ngx/src/app/modules/home/models/widget-component.models.ts index 8230fe5487..4e250b71ea 100644 --- a/ui-ngx/src/app/modules/home/models/widget-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/widget-component.models.ts @@ -78,6 +78,7 @@ import { PageLink } from '@shared/models/page/page-link'; import { SortOrder } from '@shared/models/page/sort-order'; import { DomSanitizer } from '@angular/platform-browser'; import { Router } from '@angular/router'; +import { EdgeService } from '@core/http/edge.service'; import * as RxJS from 'rxjs'; import * as RxJSOperators from 'rxjs/operators'; import { TbPopoverComponent } from '@shared/components/popover.component'; @@ -160,6 +161,7 @@ export class WidgetContext { deviceService: DeviceService; assetService: AssetService; entityViewService: EntityViewService; + edgeService: EdgeService; customerService: CustomerService; dashboardService: DashboardService; userService: UserService; From 98011203848e1b03af0e6e04c15a4ffb95a73c6d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Jun 2022 17:07:14 +0300 Subject: [PATCH 36/80] Fixed tests --- .../controller/BaseEntityRelationControllerTest.java | 2 +- .../server/common/msg/tools/RateLimitsTest.java | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java index 1a032a392b..65558e1955 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java @@ -45,7 +45,7 @@ import java.util.List; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j -public class BaseEntityRelationControllerTest extends AbstractControllerTest { +public abstract class BaseEntityRelationControllerTest extends AbstractControllerTest { public static final String BASE_DEVICE_NAME = "Test dummy device"; diff --git a/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java b/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java index c8583527f1..cf81690826 100644 --- a/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java +++ b/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java @@ -15,8 +15,11 @@ */ package org.thingsboard.server.common.msg.tools; +import org.awaitility.pollinterval.FixedPollInterval; +import org.awaitility.pollinterval.PollInterval; import org.junit.Test; +import java.time.Duration; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -39,10 +42,11 @@ public class RateLimitsTest { assertThat(rateLimits.tryConsume()).as("new token is available").isFalse(); int expectedRefillTime = (int) (((double) period / capacity) * 1000); - int gap = 100; + int gap = 500; for (int i = 0; i < capacity; i++) { await("token refill for rate limit " + rateLimitConfig) + .pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS)) .atLeast(expectedRefillTime - gap, TimeUnit.MILLISECONDS) .atMost(expectedRefillTime + gap, TimeUnit.MILLISECONDS) .untilAsserted(() -> { @@ -70,6 +74,7 @@ public class RateLimitsTest { int gap = 500; await("tokens refill for rate limit " + rateLimitConfig) + .pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS)) .atLeast(expectedRefillTime - gap, TimeUnit.MILLISECONDS) .atMost(expectedRefillTime + gap, TimeUnit.MILLISECONDS) .untilAsserted(() -> { From 40823dd2c0da480e3284a660644ff9fbcfb565f3 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 17:40:37 +0300 Subject: [PATCH 37/80] msa black box tests: redis-cluster test implemented with mvn -DblackBoxTests.redisCluster=true --- docker/cache-redis-cluster.env | 5 + docker/cache-redis.env | 2 + docker/docker-compose.hybrid.yml | 4 - docker/docker-compose.kafka.yml | 4 - docker/docker-compose.postgres.yml | 8 - docker/docker-compose.redis-cluster.yml | 162 ++++++++++++++++++ docker/docker-compose.redis.yml | 102 +++++++++++ docker/docker-compose.yml | 11 +- docker/docker-install-tb.sh | 4 +- docker/docker-remove-services.sh | 2 +- docker/docker-start-services.sh | 2 +- docker/docker-stop-services.sh | 2 +- docker/docker-update-service.sh | 4 +- docker/docker-upgrade-tb.sh | 6 +- docker/tb-coap-transport.env | 3 - docker/tb-http-transport.env | 3 - docker/tb-lwm2m-transport.env | 3 - docker/tb-mqtt-transport.env | 3 - docker/tb-node.env | 2 - docker/tb-snmp-transport.env | 3 - msa/black-box-tests/README.md | 7 +- .../server/msa/ContainerTestSuite.java | 8 +- .../server/msa/ThingsBoardDbInstaller.java | 46 ++++- 23 files changed, 337 insertions(+), 59 deletions(-) create mode 100644 docker/cache-redis-cluster.env create mode 100644 docker/cache-redis.env create mode 100644 docker/docker-compose.redis-cluster.yml create mode 100644 docker/docker-compose.redis.yml diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env new file mode 100644 index 0000000000..de666e003b --- /dev/null +++ b/docker/cache-redis-cluster.env @@ -0,0 +1,5 @@ +CACHE_TYPE=redis +REDIS_CONNECTION_TYPE=cluster +REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 +REDIS_USE_DEFAULT_POOL_CONFIG=false +REDIS_PASSWORD=bitnami \ No newline at end of file diff --git a/docker/cache-redis.env b/docker/cache-redis.env new file mode 100644 index 0000000000..52091ecc31 --- /dev/null +++ b/docker/cache-redis.env @@ -0,0 +1,2 @@ +CACHE_TYPE=redis +REDIS_HOST=redis \ No newline at end of file diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 7d7ab84c8f..e3ba32b779 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -39,7 +39,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-core2: @@ -47,7 +46,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-rule-engine1: @@ -55,7 +53,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-rule-engine2: @@ -63,6 +60,5 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 1683114605..82fbb6ed62 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -36,25 +36,21 @@ services: - queue-kafka.env depends_on: - kafka - - redis tb-core2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine1: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-mqtt-transport1: env_file: - queue-kafka.env diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index 591ea59f7a..8fe8e6f53d 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -31,27 +31,19 @@ services: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-core2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine1: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml new file mode 100644 index 0000000000..e66122f44d --- /dev/null +++ b/docker/docker-compose.redis-cluster.yml @@ -0,0 +1,162 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis cluster + redis-node-0: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-0:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-1: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-1:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-2: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-2:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-3: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-3:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-4: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-4:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-5: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-5:/bitnami/redis/data + depends_on: + - redis-node-0 + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + - 'REDISCLI_AUTH=bitnami' + - 'REDIS_CLUSTER_REPLICAS=1' + - 'REDIS_CLUSTER_CREATOR=yes' + +# ThingsBoard setup to use redis-cluster + tb-core1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-core2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-coap-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-lwm2m-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-snmp-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 +volumes: + redis-cluster_data-0: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_0} + redis-cluster_data-1: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_1} + redis-cluster_data-2: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_2} + redis-cluster_data-3: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_3} + redis-cluster_data-4: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_4} + redis-cluster_data-5: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml new file mode 100644 index 0000000000..671c1952b8 --- /dev/null +++ b/docker/docker-compose.redis.yml @@ -0,0 +1,102 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis standalone + redis: + restart: always + image: bitnami/redis:7.0 + environment: + # ALLOW_EMPTY_PASSWORD is recommended only for development. + ALLOW_EMPTY_PASSWORD: "yes" + ports: + - '6379:6379' + volumes: + - 'redis_data:/bitnami/redis/data' + +# ThingsBoard setup to use redis-standalone + tb-core1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-core2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-coap-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-lwm2m-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-snmp-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor2: + env_file: + - cache-redis.env + depends_on: + - redis + +volumes: + redis_data: + external: true + name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1ba6eda32d..62969b1051 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,15 +26,10 @@ services: environment: ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - redis: - restart: always - image: redis:4.0 - ports: - - "6379" tb-js-executor: restart: always image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" - scale: 20 + scale: 10 env_file: - tb-js-executor.env tb-core1: @@ -59,7 +54,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -85,7 +79,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -109,7 +102,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-rule-engine2: restart: always @@ -131,7 +123,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-mqtt-transport1: restart: always diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 6f6b1e9511..3ab5a873f0 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -48,9 +48,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 89f0f7844c..11bd451aa1 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 9f159774d8..e6b06718c5 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 61e68d6dd5..dd2ccf9ef9 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 739fcf6543..e6ec0c62a4 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -23,5 +23,5 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 3b1415a965..6a4311ffba 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -46,8 +46,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index 9e6a41c930..079443c98b 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 1b4ce7a298..7e0679987f 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -7,6 +7,3 @@ HTTP_REQUEST_TIMEOUT=60000 METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-lwm2m-transport.env b/docker/tb-lwm2m-transport.env index 4616d45972..f284803a46 100644 --- a/docker/tb-lwm2m-transport.env +++ b/docker/tb-lwm2m-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 0cd51c7371..e38cb2124a 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-node.env b/docker/tb-node.env index e3393d41b1..ba66757ecc 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,8 +4,6 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 JS_EVALUATOR=remote TRANSPORT_TYPE=remote -CACHE_TYPE=redis -REDIS_HOST=redis HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false diff --git a/docker/tb-snmp-transport.env b/docker/tb-snmp-transport.env index 4851e9f6c1..e2cc39d658 100644 --- a/docker/tb-snmp-transport.env +++ b/docker/tb-snmp-transport.env @@ -6,6 +6,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index 9c68789e67..6e70de17ca 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -18,8 +18,13 @@ As result, in REPOSITORY column, next images should be present: thingsboard/tb-web-ui thingsboard/tb-js-executor -- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory: +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone: mvn clean install -DblackBoxTests.skip=false +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis cluster: + + mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index f60d6d7545..c9ed8a4040 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -39,7 +39,7 @@ import static org.junit.Assert.fail; @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) @Slf4j public class ContainerTestSuite { - + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; @@ -52,6 +52,7 @@ public class ContainerTestSuite { @ClassRule public static DockerComposeContainer getTestContainer() { if (testContainer == null) { + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); try { final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; @@ -75,7 +76,10 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose.kafka.yml")) + new File(targetDir + "docker-compose.kafka.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.yml") + : new File("./../../docker/docker-compose.redis.yml")) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index de55a3afa3..ac0fa55991 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.msa; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; @@ -24,10 +25,16 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +@Slf4j public class ThingsBoardDbInstaller extends ExternalResource { + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; + private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume"; + private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume"; private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume"; private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume"; @@ -39,6 +46,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final DockerComposeExecutor dockerCompose; private final String postgresDataVolume; + + private final String redisDataVolume; + private final String redisClusterDataVolume; private final String tbLogVolume; private final String tbCoapTransportLogVolume; private final String tbLwm2mTransportLogVolume; @@ -49,14 +59,21 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final Map env; public ThingsBoardDbInstaller() { + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), new File("./../../docker/docker-compose.postgres.yml"), - new File("./../../docker/docker-compose.postgres.volumes.yml")); + new File("./../../docker/docker-compose.postgres.volumes.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.yml") + : new File("./../../docker/docker-compose.redis.yml") + ); String identifier = Base58.randomString(6).toLowerCase(); String project = identifier + Base58.randomString(6).toLowerCase(); postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; + redisDataVolume = project + "_" + REDIS_DATA_VOLUME; + redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME; tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME; tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME; @@ -76,6 +93,13 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); + } + } else { + env.put("REDIS_DATA_VOLUME", redisDataVolume); + } dockerCompose.withEnv(env); } @@ -111,7 +135,20 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); - dockerCompose.withCommand("up -d redis postgres"); + String redisService = ""; + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + redisService = redisService + " redis-node-" + i; + dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i); + dockerCompose.invokeDocker(); + } + } else { + redisService = "redis"; + dockerCompose.withCommand("volume create " + redisDataVolume); + dockerCompose.invokeDocker(); + } + + dockerCompose.withCommand("up -d postgres " + redisService); dockerCompose.invokeCompose(); dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); @@ -137,7 +174,10 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + + (IS_REDIS_CLUSTER + ? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining()) + : redisDataVolume)); dockerCompose.invokeDocker(); } From d6b516e708d449b0c82b381502f8a6bc087d2e9c Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 22 Jun 2022 18:06:50 +0300 Subject: [PATCH 38/80] VC queue message chunking --- .../impl/BaseEntityImportService.java | 1 + .../DefaultEntitiesVersionControlService.java | 18 +-- .../DefaultGitVersionControlQueueService.java | 117 ++++++++++++++---- .../src/main/resources/thingsboard.yml | 1 + common/cluster-api/src/main/proto/queue.proto | 16 ++- .../server/common/data/StringUtils.java | 6 + .../common/data/sync/vc/EntityDataDiff.java | 1 - .../data/sync/vc/VersionedEntityInfo.java | 1 - .../common/util/CollectionsUtil.java | 8 ++ .../DefaultClusterVersionControlService.java | 60 +++++++-- .../server/service/sync/vc/PendingCommit.java | 13 +- .../src/main/resources/tb-vc-executor.yml | 1 + ui-ngx/src/app/shared/models/vc.models.ts | 1 - 13 files changed, 195 insertions(+), 49 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 26c5a869f9..285401e3bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -160,6 +160,7 @@ public abstract class BaseEntityImportService importResult; try { importResult = exportImportService.importEntity(ctx, entityData); @@ -391,6 +392,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } + log.debug("Imported {} pack for tenant {}", entityType, ctx.getTenantId()); offset += limit; } while (entityDataList.size() == limit); } @@ -456,18 +458,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityId externalId = ((ExportableEntity) entity).getExternalId(); if (externalId == null) externalId = entityId; - return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), + return transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> { SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null, EntityExportSettings.builder() .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) .build()); - EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId); - return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), - JacksonUtil.toPrettyString(currentVersion.sort()), - JacksonUtil.toPrettyString(otherVersion.sort())), - rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); + EntityExportData currentVersion; + try { + currentVersion = exportImportService.exportEntity(ctx, entityId); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + return new EntityDataDiff(currentVersion.sort(), otherVersion.sort()); }, MoreExecutors.directExecutor()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 99246d282f..a1c79a2aac 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import lombok.SneakyThrows; @@ -23,6 +26,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.CollectionsUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; @@ -70,12 +74,15 @@ import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -92,9 +99,12 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final SchedulerComponent scheduler; private final Map> pendingRequestMap = new HashMap<>(); + private final Map> chunkedMsgs = new ConcurrentHashMap<>(); @Value("${queue.vc.request-timeout:60000}") private int requestTimeout; + @Value("${queue.vc.msg-chunk-size:500000}") + private int msgChunkSize; public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, DataDecodingEncodingService encodingService, @@ -114,24 +124,39 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build() - ).build(), wrap(future, commit)); + ).build(), wrap(future, commit, commit.getRequestId())); return future; } + @SuppressWarnings("UnstableApiUsage") @Override public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { - SettableFuture future = SettableFuture.create(); - String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId()); String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); - registerAndSend(commit, builder -> builder.setCommitRequest( - buildCommitRequest(commit).setAddMsg( - TransportProtos.AddMsg.newBuilder() - .setRelativePath(path).setEntityDataJson(entityDataJson).build() - ).build() - ).build(), wrap(future, null)); - return future; + Iterable entityDataChunks = StringUtils.split(entityDataJson, msgChunkSize); + String chunkedMsgId = UUID.randomUUID().toString(); + int chunksCount = Iterables.size(entityDataChunks); + + AtomicInteger chunkIndex = new AtomicInteger(); + List> futures = new ArrayList<>(); + entityDataChunks.forEach(chunk -> { + SettableFuture chunkFuture = SettableFuture.create(); + log.trace("[{}] sending chunk {} for 'addToCommit'", chunkedMsgId, chunkIndex.get()); + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setAddMsg( + TransportProtos.AddMsg.newBuilder() + .setRelativePath(path).setEntityDataJsonChunk(chunk) + .setChunkedMsgId(chunkedMsgId).setChunkIndex(chunkIndex.getAndIncrement()) + .setChunksCount(chunksCount).build() + ).build() + ).build(), wrap(chunkFuture, null, commit.getRequestId())); + futures.add(chunkFuture); + }); + return Futures.transform(Futures.allAsList(futures), r -> { + log.trace("[{}] sent all chunks for 'addToCommit'", chunkedMsgId); + return null; + }, MoreExecutors.directExecutor()); } @Override @@ -144,7 +169,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu buildCommitRequest(commit).setDeleteMsg( TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() ).build() - ).build(), wrap(future, null)); + ).build(), wrap(future, null, commit.getRequestId())); return future; } @@ -220,7 +245,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() - .setBranchName(branch) .setVersionId(versionId) .setEntityType(entityType.name()) .build()); @@ -229,7 +253,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() - .setBranchName(branch) .setVersionId(versionId) .build()); } @@ -289,9 +312,11 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var requestBody = enrichFunction.apply(newRequestProto(request, settings)); log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); - request.setTimeoutTask(scheduler.schedule(() -> { - processTimeout(request.getRequestId()); - }, requestTimeout, TimeUnit.MILLISECONDS)); + if (request.getTimeoutTask() == null) { + request.setTimeoutTask(scheduler.schedule(() -> { + processTimeout(request.getRequestId()); + }, requestTimeout, TimeUnit.MILLISECONDS)); + } } else { throw new RuntimeException("Future is already done!"); } @@ -355,15 +380,15 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public void processResponse(VersionControlResponseMsg vcResponseMsg) { UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); - PendingGitRequest request = pendingRequestMap.remove(requestId); + PendingGitRequest request = pendingRequestMap.get(requestId); if (request == null) { log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); return; } else { log.debug("[{}] processing response: {}", requestId, vcResponseMsg); - request.getTimeoutTask().cancel(true); } var future = request.getFuture(); + boolean completed = true; if (!StringUtils.isEmpty(vcResponseMsg.getError())) { future.setException(new RuntimeException(vcResponseMsg.getError())); } else { @@ -391,12 +416,39 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); ((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse)); } else if (vcResponseMsg.hasEntityContentResponse()) { - var data = vcResponseMsg.getEntityContentResponse().getData(); - ((EntityContentGitRequest) request).getFuture().set(toData(data)); + TransportProtos.EntityContentResponseMsg responseMsg = vcResponseMsg.getEntityContentResponse(); + String[] msgChunks = chunkedMsgs.computeIfAbsent(requestId, id -> new HashMap<>()) + .computeIfAbsent(responseMsg.getChunkedMsgId(), id -> new String[responseMsg.getChunksCount()]); + msgChunks[responseMsg.getChunkIndex()] = responseMsg.getData(); + log.trace("[{}] received chunk {} for 'getEntity'", responseMsg.getChunkedMsgId(), responseMsg.getChunkIndex()); + if (CollectionsUtil.countNonNull(msgChunks) == responseMsg.getChunksCount()) { + log.trace("[{}] collected all chunks for 'getEntity'", responseMsg.getChunkedMsgId()); + String data = String.join("", msgChunks); + ((EntityContentGitRequest) request).getFuture().set(toData(data)); + } else { + completed = false; + } } else if (vcResponseMsg.hasEntitiesContentResponse()) { - var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); - ((EntitiesContentGitRequest) request).getFuture() - .set(dataList.stream().map(this::toData).collect(Collectors.toList())); + TransportProtos.EntitiesContentResponseMsg responseMsg = vcResponseMsg.getEntitiesContentResponse(); + TransportProtos.EntityContentResponseMsg item = responseMsg.getItem(); + if (responseMsg.getItemsCount() > 0) { + Map chunkedItems = chunkedMsgs.computeIfAbsent(requestId, id -> new HashMap<>()); + String[] itemChunks = chunkedItems.computeIfAbsent(item.getChunkedMsgId(), id -> { + return new String[item.getChunksCount()]; + }); + itemChunks[item.getChunkIndex()] = item.getData(); + if (chunkedItems.size() == responseMsg.getItemsCount() && chunkedItems.values().stream() + .allMatch(chunks -> CollectionsUtil.countNonNull(chunks) == chunks.length)) { + ((EntitiesContentGitRequest) request).getFuture().set(chunkedItems.values().stream() + .map(chunks -> String.join("", chunks)) + .map(this::toData) + .collect(Collectors.toList())); + } else { + completed = false; + } + } else { + ((EntitiesContentGitRequest) request).getFuture().set(Collections.emptyList()); + } } else if (vcResponseMsg.hasVersionsDiffResponse()) { TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse(); List entityVersionsDiffList = diffResponse.getDiffList().stream() @@ -416,16 +468,29 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu ((ContentsDiffGitRequest) request).getFuture().set(diff); } } + if (completed) { + removePendingRequest(requestId); + } } private void processTimeout(UUID requestId) { - PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + PendingGitRequest pendingRequest = removePendingRequest(requestId); if (pendingRequest != null) { log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout); pendingRequest.getFuture().setException(new TimeoutException("Request timed out")); } } + private PendingGitRequest removePendingRequest(UUID requestId) { + PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + if (pendingRequest != null && pendingRequest.getTimeoutTask() != null) { + pendingRequest.getTimeoutTask().cancel(true); + pendingRequest.setTimeoutTask(null); + } + chunkedMsgs.remove(requestId); + return pendingRequest; + } + private PageData toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) { var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()); return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext()); @@ -458,16 +523,18 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu }; } - private static TbQueueCallback wrap(SettableFuture future, T value) { + private TbQueueCallback wrap(SettableFuture future, T value, UUID requestId) { return new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { future.set(value); + removePendingRequest(requestId); } @Override public void onFailure(Throwable t) { future.setException(t); + removePendingRequest(requestId); } }; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 07233bb241..645daa58e7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1024,6 +1024,7 @@ queue: poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}" + msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:500000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 3079978bce..6ab364012d 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -707,7 +707,10 @@ message PrepareMsg { message AddMsg { string relativePath = 1; - string entityDataJson = 2; + string entityDataJsonChunk = 2; + string chunkedMsgId = 3; + int32 chunkIndex = 4; + int32 chunksCount = 5; } message DeleteMsg { @@ -747,9 +750,8 @@ message ListVersionsResponseMsg { } message ListEntitiesRequestMsg { - string branchName = 1; - string versionId = 2; - string entityType = 3; + string versionId = 1; + string entityType = 2; } message VersionedEntityInfoProto { @@ -778,6 +780,9 @@ message EntityContentRequestMsg { message EntityContentResponseMsg { string data = 1; + string chunkedMsgId = 2; + int32 chunkIndex = 3; + int32 chunksCount = 4; } message EntitiesContentRequestMsg { @@ -788,7 +793,8 @@ message EntitiesContentRequestMsg { } message EntitiesContentResponseMsg { - repeated string data = 1; + EntityContentResponseMsg item = 1; + int32 itemsCount = 2; } message VersionsDiffRequestMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index ff3d3c0fd4..f76757f522 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data; +import com.google.common.base.Splitter; + import static org.apache.commons.lang3.StringUtils.repeat; public class StringUtils { @@ -92,4 +94,8 @@ public class StringUtils { return input.substring(0, startIndexInclusive) + obfuscatedPart + input.substring(endIndexExclusive); } + public static Iterable split(String value, int maxPartSize) { + return Splitter.fixedLength(maxPartSize).split(value); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java index 800583b0b8..dae2b99106 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java @@ -24,5 +24,4 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; public class EntityDataDiff { private EntityExportData currentVersion; private EntityExportData otherVersion; - private String rawDiff; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index fd278cde1f..ce8c91f4ae 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.EntityId; @NoArgsConstructor public class VersionedEntityInfo { private EntityId externalId; - // etc.. public VersionedEntityInfo(EntityId externalId) { this.externalId = externalId; diff --git a/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java b/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java index 7e39f9a273..adcffb8d21 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java @@ -39,4 +39,12 @@ public class CollectionsUtil { return isNotEmpty(collection) && collection.contains(element); } + public static int countNonNull(T[] array) { + int count = 0; + for (T t : array) { + if (t != null) count++; + } + return count; + } + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index b28618372e..20188d8145 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -28,6 +29,7 @@ import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.CollectionsUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; @@ -88,6 +90,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -122,6 +125,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private long packProcessingTimeout; @Value("${vc.git.io_pool_size:3}") private int ioPoolSize; + @Value("${queue.vc.msg-chunk-size:500000}") + private int msgChunkSize; //We need to manually manage the threads since tasks for particular tenant need to be processed sequentially. private final List ioThreads = new ArrayList<>(); @@ -269,19 +274,49 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe String path = getRelativePath(entityType, null); var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path) .stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList()); - var response = EntitiesContentResponseMsg.newBuilder(); - for (VersionedEntityInfo info : ids) { - var data = vcService.getFileContentAtCommit(ctx.getTenantId(), - getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); - response.addData(data); + if (!ids.isEmpty()) { + for (VersionedEntityInfo info : ids) { + var data = vcService.getFileContentAtCommit(ctx.getTenantId(), + getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); + + Iterable dataChunks = StringUtils.split(data, msgChunkSize); + String chunkedMsgId = UUID.randomUUID().toString(); + int chunksCount = Iterables.size(dataChunks); + AtomicInteger chunkIndex = new AtomicInteger(); + dataChunks.forEach(chunk -> { + EntitiesContentResponseMsg.Builder response = EntitiesContentResponseMsg.newBuilder() + .setItemsCount(ids.size()) + .setItem(EntityContentResponseMsg.newBuilder() + .setData(chunk) + .setChunkedMsgId(chunkedMsgId) + .setChunksCount(chunksCount) + .setChunkIndex(chunkIndex.getAndIncrement()) + .build()); + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); + }); + } + } else { + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse( + EntitiesContentResponseMsg.newBuilder() + .setItemsCount(0))); } - reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); } private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException { String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId()); - reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data))); + + Iterable dataChunks = StringUtils.split(data, msgChunkSize); + String chunkedMsgId = UUID.randomUUID().toString(); + int chunksCount = Iterables.size(dataChunks); + + AtomicInteger chunkIndex = new AtomicInteger(); + dataChunks.forEach(chunk -> { + log.trace("[{}] sending chunk {} for 'getEntity'", chunkedMsgId, chunkIndex.get()); + reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder() + .setData(chunk).setChunkedMsgId(chunkedMsgId).setChunksCount(chunksCount) + .setChunkIndex(chunkIndex.getAndIncrement()))); + }); } private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception { @@ -412,7 +447,16 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } private void addToCommit(VersionControlRequestCtx ctx, PendingCommit commit, AddMsg addMsg) throws IOException { - vcService.add(commit, addMsg.getRelativePath(), addMsg.getEntityDataJson()); + log.trace("[{}] received chunk {} for 'addToCommit'", addMsg.getChunkedMsgId(), addMsg.getChunkIndex()); + Map chunkedMsgs = commit.getChunkedMsgs(); + String[] msgChunks = chunkedMsgs.computeIfAbsent(addMsg.getChunkedMsgId(), id -> new String[addMsg.getChunksCount()]); + msgChunks[addMsg.getChunkIndex()] = addMsg.getEntityDataJsonChunk(); + if (CollectionsUtil.countNonNull(msgChunks) == msgChunks.length) { + log.trace("[{}] collected all chunks for 'addToCommit'", addMsg.getChunkedMsgId()); + String entityDataJson = String.join("", msgChunks); + chunkedMsgs.remove(addMsg.getChunkedMsgId()); + vcService.add(commit, addMsg.getRelativePath(), entityDataJson); + } } private void doAbortCurrentCommit(TenantId tenantId, PendingCommit current) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index ccd5fc685e..20b1991c9f 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.sync.vc; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; +import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; @Data public class PendingCommit { @@ -31,9 +33,10 @@ public class PendingCommit { private String versionName; private String authorName; - private String authorEmail; + private Map chunkedMsgs; + public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) { this.tenantId = tenantId; this.nodeId = nodeId; @@ -44,4 +47,12 @@ public class PendingCommit { this.authorEmail = authorEmail; this.workingBranch = txId.toString(); } + + public Map getChunkedMsgs() { + if (chunkedMsgs == null) { + chunkedMsgs = new ConcurrentHashMap<>(); + } + return chunkedMsgs; + } + } diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index a97c4cfd37..1344ea194a 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -168,6 +168,7 @@ queue: partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:500000}" vc: # Pool size for handling export tasks diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 98f66f74a9..aa41efba63 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -228,7 +228,6 @@ export interface RuleChainExportData extends EntityExportData { export interface EntityDataDiff { currentVersion: EntityExportData; otherVersion: EntityExportData; - rawDiff: string; } export function entityExportDataToJsonString(data: EntityExportData): string { From 7c69b7c08096de7097f39e3b254008fd56befacf Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Jun 2022 18:44:38 +0300 Subject: [PATCH 39/80] Merge PR #6785 --- .../rpc/sync/DefaultEdgeRequestsService.java | 3 +- .../DefaultTbEntityViewService.java | 28 +++++++++++++++---- .../entityView/TbEntityViewService.java | 2 +- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index dbbb6ddb1f..90dbaf3312 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -72,6 +72,7 @@ import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; +import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import java.util.ArrayList; @@ -101,7 +102,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { private DeviceService deviceService; @Autowired - private EntityViewService entityViewService; + private TbEntityViewService entityViewService; @Autowired private DeviceProfileService deviceProfileService; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java index c7d447371f..cd30276fab 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java @@ -244,16 +244,32 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen @Override public void onComponentLifecycleMsg(ComponentLifecycleMsg componentLifecycleMsg) { Map> localCacheByTenant = localCache.computeIfAbsent(componentLifecycleMsg.getTenantId(), (k) -> new ConcurrentReferenceHashMap<>()); - if (componentLifecycleMsg.getEvent() == ComponentLifecycleEvent.DELETED) { - localCacheByTenant.clear(); //we don't know which entity was mapped before deletion - } else { - EntityView entityView = entityViewService.findEntityViewById(componentLifecycleMsg.getTenantId(), new EntityViewId(componentLifecycleMsg.getEntityId().getId())); + EntityViewId entityViewId = new EntityViewId(componentLifecycleMsg.getEntityId().getId()); + deleteOldCacheValue(localCacheByTenant, entityViewId); + if (componentLifecycleMsg.getEvent() != ComponentLifecycleEvent.DELETED) { + EntityView entityView = entityViewService.findEntityViewById(componentLifecycleMsg.getTenantId(), entityViewId); if (entityView != null) { localCacheByTenant.remove(entityView.getEntityId()); } } } + private void deleteOldCacheValue(Map> localCacheByTenant, EntityViewId entityViewId) { + for (var entry : localCacheByTenant.entrySet()) { + EntityView toDelete = null; + for (EntityView view : entry.getValue()) { + if (entityViewId.equals(view.getId())) { + toDelete = view; + break; + } + } + if (toDelete != null) { + entry.getValue().remove(toDelete); + break; + } + } + } + private ListenableFuture> copyAttributesFromEntityToEntityView(EntityView entityView, String scope, Collection keys, SecurityUser user) throws ThingsboardException { EntityViewId entityId = entityView.getId(); if (keys != null && !keys.isEmpty()) { @@ -269,8 +285,8 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen long lastUpdateTs = attributeKvEntry.getLastUpdateTs(); return startTime == 0 && endTime == 0 || (endTime == 0 && startTime < lastUpdateTs) || - (startTime == 0 && endTime > lastUpdateTs) - ? true : startTime < lastUpdateTs && endTime > lastUpdateTs; + (startTime == 0 && endTime > lastUpdateTs) || + (startTime < lastUpdateTs && endTime > lastUpdateTs); }).collect(Collectors.toList()); tsSubService.saveAndNotify(entityView.getTenantId(), entityId, scope, attributes, new FutureCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java index f66113e5e4..e762877bc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java @@ -35,7 +35,7 @@ public interface TbEntityViewService extends ComponentLifecycleListener { void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException; - void delete (EntityView entity, SecurityUser user) throws ThingsboardException; + void delete (EntityView entity, SecurityUser user) throws ThingsboardException; EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer, SecurityUser user) throws ThingsboardException; From 731fb1eae893006d5c9a5b5deee527d8fb071447 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Jun 2022 19:42:00 +0300 Subject: [PATCH 40/80] Code review changes: avoid creating device actor if not needed --- .../device/DeviceActorMessageProcessor.java | 7 ------- .../rpc/processor/TelemetryEdgeProcessor.java | 11 +++-------- .../queue/DefaultTbCoreConsumerService.java | 18 ++++++++++++++++++ .../service/queue/TbCoreConsumerStats.java | 5 +++++ common/cluster-api/src/main/proto/queue.proto | 8 ++++++-- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 28c7a6a499..b0830317a7 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -418,9 +418,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { if (msg.hasUplinkNotificationMsg()) { processUplinkNotificationMsg(context, sessionInfo, msg.getUplinkNotificationMsg()); } - if (msg.hasDeviceActivity()) { - handleDeviceActivity(context, msg.getDeviceActivity()); - } callback.onSuccess(); } @@ -732,10 +729,6 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { } } - private void handleDeviceActivity(TbActorCtx context, TransportProtos.DeviceActivityProto deviceActivityProto) { - systemContext.getDeviceStateService().onDeviceActivity(tenantId, deviceId, deviceActivityProto.getLastActivityTime()); - } - void processCredentialsUpdate(TbActorMsg msg) { if (((DeviceCredentialsUpdateNotificationMsg) msg).getDeviceCredentials().getCredentialsType() == DeviceCredentialsType.LWM2M_CREDENTIALS) { sessions.forEach((k, v) -> { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java index 88ea417c24..f30111e1ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java @@ -109,21 +109,16 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { if (EntityType.DEVICE.equals(entityId.getEntityType())) { DeviceId deviceId = new DeviceId(entityId.getId()); - TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder() + TransportProtos.DeviceActivityProto deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()).build(); - - TransportProtos.DeviceActivityProto deviceActivityProto = TransportProtos.DeviceActivityProto.newBuilder() + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) .setLastActivityTime(System.currentTimeMillis()).build(); - TransportProtos.TransportToDeviceActorMsg msg = TransportProtos.TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) - .setDeviceActivity(deviceActivityProto).build(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), - TransportProtos.ToCoreMsg.newBuilder().setToDeviceActorMsg(msg).build()), null); + TransportProtos.ToCoreMsg.newBuilder().setDeviceActivityMsg(deviceActivityMsg).build()), null); } } if (entityData.hasAttributeDeleteMsg()) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d071d591a1..a3bae8bc4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -28,6 +28,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.MsgType; @@ -236,6 +237,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); if (actorMsg.isPresent()) { @@ -520,6 +524,20 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService Date: Wed, 22 Jun 2022 19:54:10 +0300 Subject: [PATCH 41/80] Introduced new counter for device activities --- .../server/service/queue/TbCoreConsumerStats.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 8f12cbb78a..11740aeb79 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -38,6 +38,7 @@ public class TbCoreConsumerStats { public static final String SUBSCRIPTION_MSGS = "subMsgs"; public static final String TO_CORE_NOTIFICATIONS = "coreNfs"; public static final String EDGE_NOTIFICATIONS = "edgeNfs"; + public static final String DEVICE_ACTIVITIES = "deviceActivity"; private final StatsCounter totalCounter; private final StatsCounter sessionEventCounter; @@ -52,6 +53,7 @@ public class TbCoreConsumerStats { private final StatsCounter subscriptionMsgCounter; private final StatsCounter toCoreNotificationsCounter; private final StatsCounter edgeNotificationsCounter; + private final StatsCounter deviceActivitiesCounter; private final List counters = new ArrayList<>(); @@ -70,6 +72,7 @@ public class TbCoreConsumerStats { this.subscriptionMsgCounter = register(statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS)); this.toCoreNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS)); this.edgeNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, EDGE_NOTIFICATIONS)); + this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES)); } private StatsCounter register(StatsCounter counter){ @@ -114,7 +117,7 @@ public class TbCoreConsumerStats { public void log(TransportProtos.DeviceActivityProto msg) { totalCounter.increment(); - deviceStateCounter.increment(); + deviceActivitiesCounter.increment(); } public void log(TransportProtos.SubscriptionMgrMsgProto msg) { From d7c750f64e60a1dd06aa9436b39cbe9fabbecd13 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 21:30:34 +0300 Subject: [PATCH 42/80] msa black box tests: redis, postgres, node volumes refactored - volumes decoupled to separate files .volumes (used only for tests) --- docker/cache-redis-cluster.env | 2 +- docker/cache-redis.env | 2 +- docker/docker-compose.postgres.volumes.yml | 61 -------------- .../docker-compose.redis-cluster.volumes.yml | 58 +++++++++++++ docker/docker-compose.redis-cluster.yml | 31 ++----- docker/docker-compose.redis.volumes.yml | 27 +++++++ docker/docker-compose.redis.yml | 7 +- docker/docker-compose.volumes.yml | 81 +++++++++++++++++++ .../server/msa/ContainerTestSuite.java | 7 +- .../server/msa/ThingsBoardDbInstaller.java | 9 ++- 10 files changed, 188 insertions(+), 97 deletions(-) create mode 100644 docker/docker-compose.redis-cluster.volumes.yml create mode 100644 docker/docker-compose.redis.volumes.yml create mode 100644 docker/docker-compose.volumes.yml diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env index de666e003b..64058ab556 100644 --- a/docker/cache-redis-cluster.env +++ b/docker/cache-redis-cluster.env @@ -2,4 +2,4 @@ CACHE_TYPE=redis REDIS_CONNECTION_TYPE=cluster REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 REDIS_USE_DEFAULT_POOL_CONFIG=false -REDIS_PASSWORD=bitnami \ No newline at end of file +REDIS_PASSWORD=bitnami diff --git a/docker/cache-redis.env b/docker/cache-redis.env index 52091ecc31..7b92620666 100644 --- a/docker/cache-redis.env +++ b/docker/cache-redis.env @@ -1,2 +1,2 @@ CACHE_TYPE=redis -REDIS_HOST=redis \ No newline at end of file +REDIS_HOST=redis diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 019e087c48..caf78f23d7 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -20,69 +20,8 @@ services: postgres: volumes: - postgres-db-volume:/var/lib/postgresql/data - tb-core1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-core2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-coap-transport: - volumes: - - tb-coap-transport-log-volume:/var/log/tb-coap-transport - tb-lwm2m-transport: - volumes: - - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport - tb-http-transport1: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-http-transport2: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-mqtt-transport1: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-mqtt-transport2: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-snmp-transport: - volumes: - - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport - tb-vc-executor1: - volumes: - - tb-vc-executor-log-volume:/var/log/tb-vc-executor - tb-vc-executor2: - volumes: - - tb-vc-executor-log-volume:/var/log/tb-vc-executor - volumes: postgres-db-volume: external: true name: ${POSTGRES_DATA_VOLUME} - tb-log-volume: - external: true - name: ${TB_LOG_VOLUME} - tb-coap-transport-log-volume: - external: true - name: ${TB_COAP_TRANSPORT_LOG_VOLUME} - tb-lwm2m-transport-log-volume: - external: true - name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} - tb-http-transport-log-volume: - external: true - name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} - tb-mqtt-transport-log-volume: - external: true - name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} - tb-snmp-transport-log-volume: - external: true - name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} - tb-vc-executor-log-volume: - external: true - name: ${TB_VC_EXECUTOR_LOG_VOLUME} \ No newline at end of file diff --git a/docker/docker-compose.redis-cluster.volumes.yml b/docker/docker-compose.redis-cluster.volumes.yml new file mode 100644 index 0000000000..2cf319bd21 --- /dev/null +++ b/docker/docker-compose.redis-cluster.volumes.yml @@ -0,0 +1,58 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + # Redis cluster + redis-node-0: + volumes: + - redis-cluster-data-0:/bitnami/redis/data + redis-node-1: + volumes: + - redis-cluster-data-1:/bitnami/redis/data + redis-node-2: + volumes: + - redis-cluster-data-2:/bitnami/redis/data + redis-node-3: + volumes: + - redis-cluster-data-3:/bitnami/redis/data + redis-node-4: + volumes: + - redis-cluster-data-4:/bitnami/redis/data + redis-node-5: + volumes: + - redis-cluster-data-5:/bitnami/redis/data + +volumes: + redis-cluster-data-0: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_0} + redis-cluster-data-1: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_1} + redis-cluster-data-2: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_2} + redis-cluster-data-3: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_3} + redis-cluster-data-4: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_4} + redis-cluster-data-5: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index e66122f44d..17ca4ed13a 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -21,7 +21,7 @@ services: redis-node-0: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-0:/bitnami/redis/data + - ./tb-node/redis-cluster-data-0:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -29,7 +29,7 @@ services: redis-node-1: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-1:/bitnami/redis/data + - ./tb-node/redis-cluster-data-1:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -37,7 +37,7 @@ services: redis-node-2: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-2:/bitnami/redis/data + - ./tb-node/redis-cluster-data-2:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -45,7 +45,7 @@ services: redis-node-3: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-3:/bitnami/redis/data + - ./tb-node/redis-cluster-data-3:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -53,7 +53,7 @@ services: redis-node-4: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-4:/bitnami/redis/data + - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -61,7 +61,7 @@ services: redis-node-5: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-5:/bitnami/redis/data + - ./tb-node/redis-cluster-data-5:/bitnami/redis/data depends_on: - redis-node-0 - redis-node-1 @@ -141,22 +141,3 @@ services: - cache-redis-cluster.env depends_on: - redis-node-5 -volumes: - redis-cluster_data-0: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_0} - redis-cluster_data-1: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_1} - redis-cluster_data-2: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_2} - redis-cluster_data-3: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_3} - redis-cluster_data-4: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_4} - redis-cluster_data-5: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis.volumes.yml b/docker/docker-compose.redis.volumes.yml new file mode 100644 index 0000000000..090aa441fe --- /dev/null +++ b/docker/docker-compose.redis.volumes.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + redis: + volumes: + - redis-data:/bitnami/redis/data + +volumes: + redis-data: + external: true + name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml index 671c1952b8..e53a974134 100644 --- a/docker/docker-compose.redis.yml +++ b/docker/docker-compose.redis.yml @@ -27,7 +27,7 @@ services: ports: - '6379:6379' volumes: - - 'redis_data:/bitnami/redis/data' + - ./tb-node/redis-data:/bitnami/redis/data # ThingsBoard setup to use redis-standalone tb-core1: @@ -95,8 +95,3 @@ services: - cache-redis.env depends_on: - redis - -volumes: - redis_data: - external: true - name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.volumes.yml b/docker/docker-compose.volumes.yml new file mode 100644 index 0000000000..58269473e4 --- /dev/null +++ b/docker/docker-compose.volumes.yml @@ -0,0 +1,81 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-core1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-core2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-coap-transport: + volumes: + - tb-coap-transport-log-volume:/var/log/tb-coap-transport + tb-lwm2m-transport: + volumes: + - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport + tb-http-transport1: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-http-transport2: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-mqtt-transport1: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-mqtt-transport2: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-snmp-transport: + volumes: + - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + +volumes: + tb-log-volume: + external: true + name: ${TB_LOG_VOLUME} + tb-coap-transport-log-volume: + external: true + name: ${TB_COAP_TRANSPORT_LOG_VOLUME} + tb-lwm2m-transport-log-volume: + external: true + name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} + tb-http-transport-log-volume: + external: true + name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} + tb-mqtt-transport-log-volume: + external: true + name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} + tb-snmp-transport-log-volume: + external: true + name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index c9ed8a4040..cc6661d8d9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -74,12 +74,17 @@ public class ContainerTestSuite { testContainer = new DockerComposeContainerImpl<>( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml")) + : new File("./../../docker/docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") + : new File("./../../docker/docker-compose.redis.volumes.yml") + ) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index ac0fa55991..08a333ab52 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -60,12 +60,17 @@ public class ThingsBoardDbInstaller extends ExternalResource { public ThingsBoardDbInstaller() { log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); - List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), + List composeFiles = Arrays.asList( + new File("./../../docker/docker-compose.yml"), + new File("./../../docker/docker-compose.volumes.yml"), new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.volumes.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml") + : new File("./../../docker/docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") + : new File("./../../docker/docker-compose.redis.volumes.yml") ); String identifier = Base58.randomString(6).toLowerCase(); From 7d7701723cc753a38f5e95a700b66b1fc2414f4e Mon Sep 17 00:00:00 2001 From: van-vanich Date: Wed, 22 Jun 2022 23:09:58 +0300 Subject: [PATCH 43/80] improve logic --- ...stractChunkedAggregationTimeseriesDao.java | 36 ++++--- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 12 --- .../dao/sqlts/AggregationTimeseriesDao.java | 29 ++++++ .../sqlts/BaseAbstractSqlTimeseriesDao.java | 23 +++++ .../dao/sqlts/SqlTimeseriesLatestDao.java | 1 - .../timescale/TimescaleTimeseriesDao.java | 5 - .../timeseries/AggregationTimeseriesDao.java | 97 ------------------- .../CassandraBaseTimeseriesDao.java | 46 +++++---- .../CassandraBaseTimeseriesLatestDao.java | 1 + ...ctChunkedAggregationTimeseriesDaoTest.java | 45 +++++---- 10 files changed, 128 insertions(+), 167 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 57b458a663..d68271798a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -26,6 +26,7 @@ import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -118,12 +119,28 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public long getTsForReadTsKvQuery(long startTs, long endTs) { - return startTs + (endTs - startTs) / 2; + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(entityId, query); + } else { + List>> futures = new ArrayList<>(); + long endPeriod = query.getEndTs(); + long startPeriod = query.getStartTs(); + long step = query.getInterval(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); + long ts = startTs + (endTs - startTs) / 2; + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, subQuery.getKey(), startTs, endTs, ts, query.getAggregation()); + futures.add(aggregateTsKvEntry); + startPeriod = endTs; + } + return getTskvEntriesFuture(Futures.allAsList(futures)); + } } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -136,17 +153,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } - @Override - public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { - return findAndAggregateAsync(entityId, - query.getKey(), - startTs, - endTs, - getTsForReadTsKvQuery(startTs, endTs), - aggregation); - - } - ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index c32123cb94..aa0d9b1c47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; -import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import javax.annotation.Nullable; import java.sql.Connection; @@ -36,7 +35,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.Objects; -import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -122,14 +120,4 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } - - @Override - public Executor getExecutor() { - return service; - } - - @Override - public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { - return interval; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..31270bacc7 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface AggregationTimeseriesDao { + + ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 83a5d11623..dd9d15b8b1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -15,18 +15,25 @@ */ package org.thingsboard.server.dao.sqlts; +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import javax.annotation.Nullable; +import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; @Slf4j public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { @@ -72,4 +79,20 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni return keyId; } + protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { + return Futures.transform(future, new Function>, List>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, service); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 7808f0e1bd..db30adead2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -45,7 +45,6 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.timeseries.AggregationTimeseriesDao; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 15875eb275..9c03a9e3dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -165,11 +165,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements super.cleanup(systemTtl); } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - return findAllAsyncWithLimit(entityId, query); - } - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java deleted file mode 100644 index b8e42e3165..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregationTimeseriesDao.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.timeseries; - -import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvEntry; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -public interface AggregationTimeseriesDao { - - default ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(tenantId, entityId, query); - } else { - List>> futures = findIntervals(tenantId, entityId, query); - return getTskvEntriesFuture(Futures.allAsList(futures)); - } - } - - default List>> findIntervals(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - List>> futures = new ArrayList<>(); - long endPeriod = query.getEndTs(); - long startPeriod = query.getStartTs(); - long step = query.getInterval(); - while (startPeriod <= endPeriod) { - long startTs = startPeriod; - long endTs = Math.min(startPeriod + step, endPeriod + 1); - long ts = getTsForReadTsKvQuery(startTs, endTs); - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); - ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs), query.getAggregation()); - futures.add(aggregateTsKvEntry); - startPeriod = endTs; - } - return futures; - } - - default long getTsForReadTsKvQuery(long startTs, long endTs) { - return endTs - startTs; - } - - long getIntervalGreaterOrEqualsMinAggregationStep(long interval); - - default ListenableFuture> getTskvEntriesFuture(ListenableFuture>> allAsList) { - return Futures.transform(allAsList, new Function<>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, getExecutor()); - } - - Executor getExecutor(); - - default ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery key, long startTs, long endTs, Aggregation aggregation) { - return Futures.immediateFuture(null); - } - - ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); - - default long toPartitionTs(long ts) { - return ts; - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 8b58e547c9..bb3930b99c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -37,6 +37,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.KvEntry; @@ -45,6 +46,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -59,7 +61,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -265,13 +266,31 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public long getIntervalGreaterOrEqualsMinAggregationStep(long interval) { - return Math.max(interval, MIN_AGGREGATION_STEP_MS); - } - - @Override - public ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long startTs, long endTs, Aggregation aggregation) { - return findAndAggregateAsync(tenantId, entityId, query, startTs, endTs); + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(tenantId, entityId, query); + } else { + long startPeriod = query.getStartTs(); + long endPeriod = query.getEndTs(); + long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); + List>> futures = new ArrayList<>(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); + long ts = endTs - startTs; + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); + futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); + startPeriod = endTs; + } + ListenableFuture>> future = Futures.allAsList(futures); + return Futures.transform(future, new Function<>() { + @Nullable + @Override + public List apply(@Nullable List> input) { + return input == null ? Collections.emptyList() : input.stream().filter(v -> v.isPresent()).map(v -> v.get()).collect(Collectors.toList()); + } + }, readResultsProcessingExecutor); + } } @Override @@ -279,8 +298,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD //Cleanup by TTL is native for Cassandra } - @Override - public ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); @@ -302,17 +320,11 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return resultFuture; } - @Override - public long toPartitionTs(long ts) { + private long toPartitionTs(long ts) { LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } - @Override - public Executor getExecutor() { - return readResultsProcessingExecutor; - } - private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SimpleListenableFuture> resultFuture) { if (cursor.isFull() || !cursor.hasNextPartition()) { resultFuture.set(cursor.getData()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index e7099db3d2..124af03a1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java index cab869a647..db216f7f6e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -59,9 +59,9 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 2001, COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 2001, 3000 + 1, COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT); } @Test @@ -70,8 +70,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000 + 1, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -81,9 +81,9 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3000, COUNT); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuerySecond, 3000, 3001, COUNT); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT); } @@ -93,8 +93,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -103,8 +103,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 0, 1, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 0, 1, getTsForReadTsKvQuery(0, 1), COUNT); } @Test @@ -113,8 +113,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQuery, 1, 2, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuery.getKey(), 1, 2, getTsForReadTsKvQuery(1, 2), COUNT); } @Test @@ -123,8 +123,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), Integer.MAX_VALUE, 1L + Integer.MAX_VALUE, getTsForReadTsKvQuery(Integer.MAX_VALUE, 1L + Integer.MAX_VALUE), COUNT); } @Test @@ -133,8 +133,8 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, subQueryFirst, 1, 3001, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); } @Test @@ -142,10 +142,15 @@ public class AbstractChunkedAggregationTimeseriesDaoTest { ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC); willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); - verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), any(), anyLong(), anyLong(), any()); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); for (long i = 1; i <= 3000; i += 3) { ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); - verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, SYS_TENANT_ID, querySub, i, i + 3, COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT); } } + + long getTsForReadTsKvQuery(long startTs, long endTs) { + return startTs + (endTs - startTs) / 2L; + } + } From 7b88a437dc194eaab7dc0f2aeebf7dd6d45e2707 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 23:24:05 +0300 Subject: [PATCH 44/80] docker-compose shell scripts: added CACHE=redis or CACHE=redis-cluster option. Default is redis, README.md updated --- docker/.env | 3 +++ docker/README.md | 7 ++++++ docker/compose-utils.sh | 36 +++++++++++++++++++++++++++-- docker/docker-create-log-folders.sh | 21 +++++++++++++++++ docker/docker-install-tb.sh | 6 +++-- docker/docker-remove-services.sh | 4 +++- docker/docker-start-services.sh | 4 +++- docker/docker-stop-services.sh | 4 +++- docker/docker-update-service.sh | 6 +++-- docker/docker-upgrade-tb.sh | 8 ++++--- 10 files changed, 87 insertions(+), 12 deletions(-) diff --git a/docker/.env b/docker/.env index af0f537603..7bc1a7a6a4 100644 --- a/docker/.env +++ b/docker/.env @@ -1,5 +1,8 @@ TB_QUEUE_TYPE=kafka +# redis or redis-cluster +CACHE=redis + DOCKER_REPO=thingsboard JS_EXECUTOR_DOCKER_NAME=tb-js-executor diff --git a/docker/README.md b/docker/README.md index 01f89ebb6c..71ea87f353 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env` **NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details). +In order to set cache type change the value of `CACHE` variable in `.env` file to one of the following: + +- `redis` - use Redis standalone cache (1 node - 1 master); +- `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves); + +**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details). + Execute the following command to create log folders for the services and chown of these folders to the docker container users. To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access): diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 550f2dfea4..d60c81ec6a 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -73,19 +73,51 @@ function additionalComposeMonitoringArgs() { fi } +function additionalComposeCacheArgs() { + source .env + CACHE_COMPOSE_ARGS="" + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + CACHE_COMPOSE_ARGS="-f docker-compose.redis.yml" + ;; + redis-cluster) + CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 + esac + echo $CACHE_COMPOSE_ARGS +} + function additionalStartupServices() { source .env ADDITIONAL_STARTUP_SERVICES="" case $DATABASE in postgres) - ADDITIONAL_STARTUP_SERVICES=postgres + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres" ;; hybrid) - ADDITIONAL_STARTUP_SERVICES="postgres cassandra" + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra" ;; *) echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 exit 1 esac + + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis" + ;; + redis-cluster) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 + esac + echo $ADDITIONAL_STARTUP_SERVICES } diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index ba945a19df..83257c0e66 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -28,3 +28,24 @@ mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log + +mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/ + +source .env +CACHE="${CACHE:-redis}" +case $CACHE in + redis) + mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/ + ;; + redis-cluster) + mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/ + mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/ + mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/ + mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/ + mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/ + mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/ + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 +esac \ No newline at end of file diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 3ab5a873f0..27e84c4d0b 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -45,12 +45,14 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 11bd451aa1..36e464f45d 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index e6b06718c5..7993ab425b 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index dd2ccf9ef9..ae8a3c0b81 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index e6ec0c62a4..a02f9e208a 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -23,5 +23,7 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 6a4311ffba..c20543783b 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -44,10 +44,12 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 From 37d6cd915e34767a2d52a29a0320677e8da5a0bf Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 22 Jun 2022 18:54:55 +0300 Subject: [PATCH 45/80] Always add default branch to listBranches response --- .../EntitiesVersionControlController.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 11d64b0be4..dc2bac8947 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -490,15 +490,20 @@ public class EntitiesVersionControlController extends BaseController { List infos = new ArrayList<>(); String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); - if (StringUtils.isNotEmpty(defaultBranch)) { - infos.add(new BranchInfo(defaultBranch, true)); + if (StringUtils.isEmpty(defaultBranch)) { + if (remoteBranches.contains("main")) { + defaultBranch = "main"; + } else { + defaultBranch = "master"; + } } + infos.add(new BranchInfo(defaultBranch, true)); - remoteBranches.forEach(branch -> { + for (String branch : remoteBranches) { if (!branch.equals(defaultBranch)) { infos.add(new BranchInfo(branch, false)); } - }); + } return infos; }, MoreExecutors.directExecutor())); } From db43c39c545fb3093d746ff9ade86f82ca41f529 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 23 Jun 2022 10:44:07 +0300 Subject: [PATCH 46/80] Fix device version load --- .../service/sync/vc/DefaultEntitiesVersionControlService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 6ea8115e53..76f5bc2901 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -358,6 +358,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) + .saveCredentials(config.isLoadCredentials()) .findExistingByName(config.isFindExistingEntityByName()) .build(); } From d1995bc27ee9d7ed4654178969b12acd4b8015e3 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Thu, 23 Jun 2022 12:22:09 +0300 Subject: [PATCH 47/80] UI: Layout for tenant profile and queue --- .../entity/add-entity-dialog.component.html | 2 +- .../profile/tenant-profile.component.html | 6 +- .../profile/tenant-profile.component.scss | 4 + ...enant-profile-configuration.component.html | 876 ++++++++++-------- ...enant-profile-configuration.component.scss | 55 ++ ...-tenant-profile-configuration.component.ts | 4 +- .../queue/queue-form.component.html | 274 +++--- .../queue/queue-form.component.scss | 32 +- .../assets/locale/locale.constant-en_US.json | 184 ++-- 9 files changed, 837 insertions(+), 600 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss diff --git a/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.html b/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.html index 2614b6e8cf..84eacb4307 100644 --- a/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/add-entity-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ translations.add }}

diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html index fb20387608..e34bb8cad3 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html @@ -47,8 +47,8 @@
-
- +
+ tenant-profile.name @@ -99,7 +99,7 @@
- + tenant-profile.description diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.scss b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.scss index f9940b3dae..fb36470cee 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.scss +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.scss @@ -14,6 +14,10 @@ * limitations under the License. */ :host ::ng-deep { + .mat-form-field-infix { + width: 100% !important; + } + .mat-checkbox{ &.hinted-checkbox { .mat-checkbox-inner-container { diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 3712bcb21d..770f60067d 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -16,387 +16,499 @@ -->
- - tenant-profile.maximum-devices - - - {{ 'tenant-profile.maximum-devices-required' | translate}} - - - {{ 'tenant-profile.maximum-devices-range' | translate}} - - - - tenant-profile.maximum-assets - - - {{ 'tenant-profile.maximum-assets-required' | translate}} - - - {{ 'tenant-profile.maximum-assets-range' | translate}} - - - - tenant-profile.maximum-customers - - - {{ 'tenant-profile.maximum-customers-required' | translate}} - - - {{ 'tenant-profile.maximum-customers-range' | translate}} - - - - tenant-profile.maximum-users - - - {{ 'tenant-profile.maximum-users-required' | translate}} - - - {{ 'tenant-profile.maximum-users-range' | translate}} - - - - tenant-profile.maximum-dashboards - - - {{ 'tenant-profile.maximum-dashboards-required' | translate}} - - - {{ 'tenant-profile.maximum-dashboards-range' | translate}} - - - - tenant-profile.maximum-rule-chains - - - {{ 'tenant-profile.maximum-rule-chains-required' | translate}} - - - {{ 'tenant-profile.maximum-rule-chains-range' | translate}} - - - - tenant-profile.maximum-resources-sum-data-size - - - {{ 'tenant-profile.maximum-resources-sum-data-size-required' | translate}} - - - {{ 'tenant-profile.maximum-resources-sum-data-size-range' | translate}} - - - - tenant-profile.maximum-ota-packages-sum-data-size - - - {{ 'tenant-profile.maximum-ota-packages-sum-data-size-required' | translate}} - - - {{ 'tenant-profile.maximum-ota-packages-sum-data-size-range' | translate}} - - - - tenant-profile.max-transport-messages - - - {{ 'tenant-profile.max-transport-messages-range' | translate}} - - - {{ 'tenant-profile.max-transport-messages-required' | translate}} - - - - tenant-profile.max-transport-data-points - - - {{ 'tenant-profile.max-transport-data-points-required' | translate}} - - - {{ 'tenant-profile.max-transport-data-points-range' | translate}} - - - - tenant-profile.max-r-e-executions - - - {{ 'tenant-profile.max-r-e-executions-required' | translate}} - - - {{ 'tenant-profile.max-r-e-executions-range' | translate}} - - - - tenant-profile.max-j-s-executions - - - {{ 'tenant-profile.max-j-s-executions-required' | translate}} - - - {{ 'tenant-profile.max-j-s-executions-range' | translate}} - - - - tenant-profile.max-d-p-storage-days - - - {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} - - - {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} - - - - tenant-profile.default-storage-ttl-days - - - {{ 'tenant-profile.default-storage-ttl-days-required' | translate}} - - - {{ 'tenant-profile.default-storage-ttl-days-range' | translate}} - - - - tenant-profile.alarms-ttl-days - - - {{ 'tenant-profile.alarms-ttl-days-required' | translate}} - - - {{ 'tenant-profile.alarms-ttl-days-days-range' | translate}} - - - - tenant-profile.rpc-ttl-days - - - {{ 'tenant-profile.rpc-ttl-days-required' | translate}} - - - {{ 'tenant-profile.rpc-ttl-days-days-range' | translate}} - - - - tenant-profile.max-rule-node-executions-per-message - - - {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} - - - {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} - - - - tenant-profile.max-emails - - - {{ 'tenant-profile.max-emails-required' | translate}} - - - {{ 'tenant-profile.max-emails-range' | translate}} - - - - tenant-profile.max-sms - - - {{ 'tenant-profile.max-sms-required' | translate}} - - - {{ 'tenant-profile.max-sms-range' | translate}} - - - - tenant-profile.max-created-alarms - - - {{ 'tenant-profile.max-created-alarms-required' | translate}} - - - {{ 'tenant-profile.max-created-alarms-range' | translate}} - - +
+ {{ 'tenant-profile.entities' | translate }} (0 - unlimited) +
+ + tenant-profile.maximum-devices + + + {{ 'tenant-profile.maximum-devices-required' | translate}} + + + {{ 'tenant-profile.maximum-devices-range' | translate}} + + + + tenant-profile.maximum-dashboards + + + {{ 'tenant-profile.maximum-dashboards-required' | translate}} + + + {{ 'tenant-profile.maximum-dashboards-range' | translate}} + + +
+
+ + tenant-profile.maximum-assets + + + {{ 'tenant-profile.maximum-assets-required' | translate}} + + + {{ 'tenant-profile.maximum-assets-range' | translate}} + + + + tenant-profile.maximum-users + + + {{ 'tenant-profile.maximum-users-required' | translate}} + + + {{ 'tenant-profile.maximum-users-range' | translate}} + + +
+ + + + tenant-profile.advanced-settings + + + +
+ + tenant-profile.maximum-customers + + + {{ 'tenant-profile.maximum-customers-required' | translate}} + + + {{ 'tenant-profile.maximum-customers-range' | translate}} + + + + tenant-profile.maximum-rule-chains + + + {{ 'tenant-profile.maximum-rule-chains-required' | translate}} + + + {{ 'tenant-profile.maximum-rule-chains-range' | translate}} + + +
+ + +
- - tenant-profile.transport-tenant-msg-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-tenant-telemetry-msg-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-tenant-telemetry-data-points-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-device-msg-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.transport-device-telemetry-msg-rate-limit - - - - tenant-profile.transport-device-telemetry-data-points-rate-limit - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.tenant-rest-limits - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.customer-rest-limits - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.ws-limit-max-sessions-per-tenant - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-sessions-per-customer - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-sessions-per-regular-user - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-sessions-per-public-user - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-queue-per-session - - - {{ 'tenant-profile.too-small-value-one' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-tenant - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-customer - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-regular-user - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-max-subscriptions-per-public-user - - - {{ 'tenant-profile.too-small-value-zero' | translate}} - - - - tenant-profile.ws-limit-updates-per-session - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.cassandra-tenant-limits-configuration - - - {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} - - - - tenant-profile.tenant-entity-export-rate-limit - - - - tenant-profile.tenant-entity-import-rate-limit - - +
+ {{ 'tenant-profile.rule-engine' | translate }} (0 - unlimited) +
+ + tenant-profile.max-r-e-executions + + + {{ 'tenant-profile.max-r-e-executions-required' | translate}} + + + {{ 'tenant-profile.max-r-e-executions-range' | translate}} + + + + tenant-profile.max-transport-messages + + + {{ 'tenant-profile.max-transport-messages-range' | translate}} + + + {{ 'tenant-profile.max-transport-messages-required' | translate}} + + +
+ + + + tenant-profile.advanced-settings + + + +
+ + tenant-profile.max-j-s-executions + + + {{ 'tenant-profile.max-j-s-executions-required' | translate}} + + + {{ 'tenant-profile.max-j-s-executions-range' | translate}} + + + + tenant-profile.max-transport-data-points + + + {{ 'tenant-profile.max-transport-data-points-required' | translate}} + + + {{ 'tenant-profile.max-transport-data-points-range' | translate}} + + +
+
+ + tenant-profile.max-rule-node-executions-per-message + + + {{ 'tenant-profile.max-rule-node-executions-per-message-required' | translate}} + + + {{ 'tenant-profile.max-rule-node-executions-per-message-range' | translate}} + + +
+
+ + +
+ +
+ {{ 'tenant-profile.time-to-live' | translate }} (0 - unlimited) +
+ + tenant-profile.max-d-p-storage-days + + + {{ 'tenant-profile.max-d-p-storage-days-required' | translate}} + + + {{ 'tenant-profile.max-d-p-storage-days-range' | translate}} + + + + tenant-profile.alarms-ttl-days + + + {{ 'tenant-profile.alarms-ttl-days-required' | translate}} + + + {{ 'tenant-profile.alarms-ttl-days-days-range' | translate}} + + +
+
+ + tenant-profile.default-storage-ttl-days + + + {{ 'tenant-profile.default-storage-ttl-days-required' | translate}} + + + {{ 'tenant-profile.default-storage-ttl-days-range' | translate}} + + + + tenant-profile.rpc-ttl-days + + + {{ 'tenant-profile.rpc-ttl-days-required' | translate}} + + + {{ 'tenant-profile.rpc-ttl-days-days-range' | translate}} + + +
+
+ +
+ {{ 'tenant-profile.alarms-and-notifications' | translate }} (0 - unlimited) +
+ + tenant-profile.max-emails + + + {{ 'tenant-profile.max-emails-required' | translate}} + + + {{ 'tenant-profile.max-emails-range' | translate}} + + + + tenant-profile.max-created-alarms + + + {{ 'tenant-profile.max-created-alarms-required' | translate}} + + + {{ 'tenant-profile.max-created-alarms-range' | translate}} + + +
+
+ + tenant-profile.max-sms + + + {{ 'tenant-profile.max-sms-required' | translate}} + + + {{ 'tenant-profile.max-sms-range' | translate}} + + +
+
+
+ +
+ {{ 'tenant-profile.ota-files-in-bytes' | translate }} (0 - unlimited) +
+ + tenant-profile.maximum-resources-sum-data-size + + + {{ 'tenant-profile.maximum-resources-sum-data-size-required' | translate}} + + + {{ 'tenant-profile.maximum-resources-sum-data-size-range' | translate}} + + + + tenant-profile.maximum-ota-packages-sum-data-size + + + {{ 'tenant-profile.maximum-ota-packages-sum-data-size-required' | translate}} + + + {{ 'tenant-profile.maximum-ota-packages-sum-data-size-range' | translate}} + + +
+
+ +
+ {{ 'tenant-profile.ws-title' | translate }} (0 - unlimited) +
+ + tenant-profile.ws-limit-max-sessions-per-tenant + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-tenant + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + +
+
+ + tenant-profile.ws-limit-max-sessions-per-customer + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-customer + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + +
+ + + + tenant-profile.advanced-settings + + + +
+ + tenant-profile.ws-limit-max-sessions-per-public-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-public-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + +
+
+ + tenant-profile.ws-limit-max-sessions-per-regular-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + + + tenant-profile.ws-limit-max-subscriptions-per-regular-user + + + {{ 'tenant-profile.too-small-value-zero' | translate}} + + +
+
+ + tenant-profile.ws-limit-queue-per-session + + + {{ 'tenant-profile.too-small-value-one' | translate}} + + +
+
+ + +
+ +
+ Rate limits +
+ + tenant-profile.transport-tenant-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + +
+
+ + tenant-profile.transport-tenant-telemetry-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-telemetry-msg-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + +
+ + + + tenant-profile.advanced-settings + + + +
+ + tenant-profile.transport-tenant-telemetry-data-points-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.transport-device-telemetry-data-points-rate-limit + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + +
+
+ + tenant-profile.tenant-rest-limits + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.customer-rest-limits + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + +
+
+ + tenant-profile.tenant-entity-export-rate-limit + + + + tenant-profile.tenant-entity-import-rate-limit + + +
+
+ + tenant-profile.ws-limit-updates-per-session + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + + + tenant-profile.cassandra-tenant-limits-configuration + + + {{ 'tenant-profile.incorrect-pattern-for-rate-limits' | translate}} + + +
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss new file mode 100644 index 0000000000..d6ad9f6a37 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +:host { + .fields-element { + padding-top: 15px; + } + + .group-title > span { + color: rgba(0, 0, 0, 0.54); + } + + ::ng-deep { + .mat-expansion-panel { + &.configuration-panel { + box-shadow: none; + + .mat-expansion-panel-header { + padding: 0; + + &:hover { + background: none !important; + } + + .mat-expansion-indicator { + padding: 2px; + } + } + + > .mat-expansion-panel-content { + > .mat-expansion-panel-body { + padding: 0; + } + } + + .mat-expansion-panel-header-description { + align-items: center; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 2f922faaac..65f030a3a6 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -25,7 +25,7 @@ import { isDefinedAndNotNull } from '@core/utils'; @Component({ selector: 'tb-default-tenant-profile-configuration', templateUrl: './default-tenant-profile-configuration.component.html', - styleUrls: [], + styleUrls: ['./default-tenant-profile-configuration.component.scss'], providers: [{ provide: NG_VALUE_ACCESSOR, useExisting: forwardRef(() => DefaultTenantProfileConfigurationComponent), @@ -35,7 +35,7 @@ import { isDefinedAndNotNull } from '@core/utils'; export class DefaultTenantProfileConfigurationComponent implements ControlValueAccessor, OnInit { defaultTenantProfileConfigurationFormGroup: FormGroup; - rateLimitsPattern = '^(((\\d+):(\\d+)),)*((\\d+):(\\d+))$'; + rateLimitsPattern = '([1-9]\\d*:[1-9]\\d*)(,[1-9]\\d*:[1-9]\\d*)*'; private requiredValue: boolean; get required(): boolean { diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html index fce4a9e499..6c34e0111f 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html @@ -17,7 +17,7 @@ --> - + admin.queue-name @@ -27,151 +27,181 @@ {{ 'queue.name-unique' | translate }} - - queue.poll-interval - - - {{ 'queue.poll-interval-required' | translate }} - - - {{ 'queue.poll-interval-min-value' | translate }} - - - - queue.partitions - - - {{ 'queue.partitions-required' | translate }} - - - {{ 'queue.partitions-min-value' | translate }} - - - -
{{ 'queue.consumer-per-partition' | translate }}
-
{{'queue.consumer-per-partition-hint' | translate}}
- - - queue.processing-timeout - - - {{ 'queue.pack-processing-timeout-required' | translate }} - - - {{ 'queue.pack-processing-timeout-min-value' | translate }} - - - + - - queue.submit-strategy + + queue.submit-settings -
- - queue.submit-strategy - - +
+
+ + + {{ queueSubmitStrategyTypesMap.get(queueSubmitStrategyTypes[strategy]).label | translate }} - - - - {{ 'queue.submit-strategy-type-required' | translate }} - - - - queue.batch-size - - - {{ 'queue.batch-size-required' | translate }} - - - {{ 'queue.batch-size-min-value' | translate }} - - + + + + +
+
+ + + queue.batch-size + + + {{ 'queue.batch-size-required' | translate }} + + + {{ 'queue.batch-size-min-value' | translate }} + + +
- + - - queue.processing-strategy + + queue.processing-settings -
- - queue.processing-strategy - - - {{ queueProcessingStrategyTypesMap.get(queueProcessingStrategyTypes[strategy]).label | translate }} - - - - {{ 'queue.processing-strategy-type-required' | translate }} - - - - queue.retries - - - {{ 'queue.retries-required' | translate }} - - +
+ + + + {{ queueProcessingStrategyTypesMap.get(queueProcessingStrategyTypes[strategy]).label | translate }} + + + + +
+
+ + + queue.retries + + + {{ 'queue.retries-required' | translate }} + + - {{ 'queue.retries-min-value' | translate }} - - - - queue.failure-percentage - - - {{ 'queue.failure-percentage-required' | translate }} - - + queue.failure-percentage + + + {{ 'queue.failure-percentage-required' | translate }} + + - {{ 'queue.failure-percentage-min-value' | translate }} - - - {{ 'queue.failure-percentage-max-value' | translate }} - - - - queue.pause-between-retries - - - {{ 'queue.pause-between-retries-required' | translate }} - - + queue.pause-between-retries + + + {{ 'queue.pause-between-retries-required' | translate }} + + - {{ 'queue.pause-between-retries-min-value' | translate }} - - - - queue.max-pause-between-retries - - - {{ 'queue.max-pause-between-retries-required' | translate }} - - + queue.max-pause-between-retries + + + {{ 'queue.max-pause-between-retries-required' | translate }} + + - {{ 'queue.max-pause-between-retries-min-value' | translate }} - - + {{ 'queue.max-pause-between-retries-min-value' | translate }} + + +
+
+ + + + + + queue.polling-settings + + + +
+
+ +
+ + queue.poll-interval + + + {{ 'queue.poll-interval-required' | translate }} + + + {{ 'queue.poll-interval-min-value' | translate }} + + + + queue.partitions + + + {{ 'queue.partitions-required' | translate }} + + + {{ 'queue.partitions-min-value' | translate }} + + +
+
+
+ +
+ + {{ 'queue.consumer-per-partition' | translate }} + + + queue.processing-timeout + + + {{ 'queue.pack-processing-timeout-required' | translate }} + + + {{ 'queue.pack-processing-timeout-min-value' | translate }} + + +
+
- + queue.description queue.description-hint diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.scss b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.scss index 13128ad29a..874660cbdf 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.scss +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.scss @@ -13,16 +13,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -:host ::ng-deep { +:host { .queue-strategy { padding-bottom: 16px; - .mat-expansion-panel-header { - height: 50px; + .group-label { + display: block; + padding-bottom: 10px; } - .mat-expansion-panel-body { - padding-bottom: 0 !important; + .panel-title { + font-weight: 500; + } + + .hint-icon { + width: 15px; + height: 15px; + vertical-align: top; + } + } + + ::ng-deep { + .mat-form-field-infix { + width: 100% !important; + } + .queue-strategy { + .mat-expansion-panel-header { + height: 50px; + } + + .mat-expansion-panel-body { + padding-bottom: 0 !important; + } } } } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 30c4d78059..cc7b296495 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2957,18 +2957,25 @@ "add" : "Add queue", "details": "Queue details", "topic": "Topic", - "submit-strategy": "Submit Strategy", - "processing-strategy": "Processing Strategy", + "submit-settings": "Submit settings", + "submit-strategy": "Strategy type *", + "grouping-parameter": "Grouping parameter", + "processing-settings": "Retries processing settings", + "processing-strategy": "Processing type *", + "retries-settings": "Retries settings", + "polling-settings": "Polling settings", + "batch-processing": "Batch processing", "poll-interval": "Poll interval", "partitions": "Partitions", - "consumer-per-partition": "Consumer per partition", + "immediate-processing": "Immediate processing", + "consumer-per-partition": "Send message poll for each consumer", "consumer-per-partition-hint": "Enable separate consumer(s) per each partition", - "processing-timeout": "Processing timeout, ms", + "processing-timeout": "Processing within, ms", "batch-size": "Batch size", - "retries": "Retries (0 - unlimited)", - "failure-percentage": "Failure Percentage", - "pause-between-retries": "Pause between retries", - "max-pause-between-retries": "Maximal pause between retries", + "retries": "Number of retries (0 – unlimited)", + "failure-percentage": "Percentage of failure messages for skipping retries", + "pause-between-retries": "Retry within, sec", + "max-pause-between-retries": "Additional retry within, sec", "delete": "Delete queue", "copyId": "Copy queue Id", "idCopiedMessage": "Queue Id has been copied to clipboard", @@ -3083,93 +3090,100 @@ "export-failed-error": "Unable to export tenant profile: {{error}}", "tenant-profile-file": "Tenant profile file", "invalid-tenant-profile-file-error": "Unable to import tenant profile: Invalid tenant profile data structure.", - "maximum-devices": "Maximum number of devices (0 - unlimited)", - "maximum-devices-required": "Maximum number of devices is required.", - "maximum-devices-range": "Minimum number of devices can't be negative", - "maximum-assets": "Maximum number of assets (0 - unlimited)", - "maximum-assets-required": "Maximum number of assets is required.", - "maximum-assets-range": "Maximum number of assets can't be negative", - "maximum-customers": "Maximum number of customers (0 - unlimited)", - "maximum-customers-required": "Maximum number of customers is required.", - "maximum-customers-range": "Maximum number of customers can't be negative", - "maximum-users": "Maximum number of users (0 - unlimited)", - "maximum-users-required": "Maximum number of users is required.", - "maximum-users-range": "Maximum number of users can't be negative", - "maximum-dashboards": "Maximum number of dashboards (0 - unlimited)", - "maximum-dashboards-required": "Maximum number of dashboards is required.", - "maximum-dashboards-range": "Maximum number of dashboards can't be negative", - "maximum-rule-chains": "Maximum number of rule chains (0 - unlimited)", - "maximum-rule-chains-required": "Maximum number of rule chains is required.", - "maximum-rule-chains-range": "Maximum number of rule chains can't be negative", - "maximum-resources-sum-data-size": "Maximum sum of resource files size in bytes (0 - unlimited)", - "maximum-resources-sum-data-size-required": "Maximum sum of resource files size is required.", - "maximum-resources-sum-data-size-range": "Maximum sum of resource files size can`t be negative", - "maximum-ota-packages-sum-data-size": "Maximum sum of ota package files size in bytes (0 - unlimited)", - "maximum-ota-package-sum-data-size-required": "Maximum sum of ota package files size is required.", - "maximum-ota-package-sum-data-size-range": "Maximum sum of ota package files size can`t be negative", - "transport-tenant-msg-rate-limit": "Transport tenant messages rate limit.", - "transport-tenant-telemetry-msg-rate-limit": "Transport tenant telemetry messages rate limit.", - "transport-tenant-telemetry-data-points-rate-limit": "Transport tenant telemetry data points rate limit.", - "transport-device-msg-rate-limit": "Transport device messages rate limit.", - "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", - "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", - "tenant-entity-export-rate-limit": "Entity version creation rate limit", - "tenant-entity-import-rate-limit": "Entity version load rate limit", - "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", - "max-transport-messages-required": "Maximum number of transport messages is required.", - "max-transport-messages-range": "Maximum number of transport messages can't be negative", - "max-transport-data-points": "Maximum number of transport data points (0 - unlimited)", - "max-transport-data-points-required": "Maximum number of transport data points is required.", - "max-transport-data-points-range": "Maximum number of transport data points can't be negative", - "max-r-e-executions": "Maximum number of Rule Engine executions (0 - unlimited)", - "max-r-e-executions-required": "Maximum number of Rule Engine executions is required.", - "max-r-e-executions-range": "Maximum number of Rule Engine executions can't be negative", - "max-j-s-executions": "Maximum number of JavaScript executions (0 - unlimited)", - "max-j-s-executions-required": "Maximum number of JavaScript executions is required.", - "max-j-s-executions-range": "Maximum number of JavaScript executions can't be negative", - "max-d-p-storage-days": "Maximum number of data points storage days (0 - unlimited)", - "max-d-p-storage-days-required": "Maximum number of data points storage days is required.", - "max-d-p-storage-days-range": "Maximum number of data points storage days can't be negative", - "default-storage-ttl-days": "Default storage TTL days (0 - unlimited)", - "default-storage-ttl-days-required": "Default storage TTL days is required.", - "default-storage-ttl-days-range": "Default storage TTL days can't be negative", - "alarms-ttl-days": "Alarms TTL days (0 - unlimited)", + "advanced-settings": "Advanced settings", + "entities": "Entities", + "rule-engine": "Rule Engine", + "time-to-live": "Time-to-live", + "alarms-and-notifications": "Alarms and notifications", + "ota-files-in-bytes": "OTA files in bytes", + "ws-title": "WS", + "maximum-devices": "Devices maximum number", + "maximum-devices-required": "Devices maximum number is required.", + "maximum-devices-range": "Devices maximum number can't be negative", + "maximum-assets": "Assets maximum number", + "maximum-assets-required": "Assets maximum number is required.", + "maximum-assets-range": "Assets maximum number can't be negative", + "maximum-customers": "Customers maximum number", + "maximum-customers-required": "Customers maximum number is required.", + "maximum-customers-range": "Customers maximum number can't be negative", + "maximum-users": "Users maximum number", + "maximum-users-required": "Users maximum number is required.", + "maximum-users-range": "Users maximum number can't be negative", + "maximum-dashboards": "Dashboards maximum number", + "maximum-dashboards-required": "Dashboards maximum number is required.", + "maximum-dashboards-range": "Dashboards maximum number can't be negative", + "maximum-rule-chains": "Rule chains maximum number", + "maximum-rule-chains-required": "Rule chains maximum number is required.", + "maximum-rule-chains-range": "Rule chains maximum number can't be negative", + "maximum-resources-sum-data-size": "Resource files sum size", + "maximum-resources-sum-data-size-required": "Resource files sum size is required.", + "maximum-resources-sum-data-size-range": "Resource files sum size can`t be negative", + "maximum-ota-packages-sum-data-size": "OTA package files sum size", + "maximum-ota-package-sum-data-size-required": "OTA package files sum size is required.", + "maximum-ota-package-sum-data-size-range": "OTA package files sum size can`t be negative", + "transport-tenant-msg-rate-limit": "Transport tenant messages", + "transport-tenant-telemetry-msg-rate-limit": "Transport tenant telemetry messages", + "transport-tenant-telemetry-data-points-rate-limit": "Transport tenant telemetry data points", + "transport-device-msg-rate-limit": "Transport device messages", + "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages", + "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points", + "tenant-entity-export-rate-limit": "Entity version creation", + "tenant-entity-import-rate-limit": "Entity version load", + "max-transport-messages": "Transport messages maximum number", + "max-transport-messages-required": "Transport messages maximum number is required.", + "max-transport-messages-range": "Transport messages maximum number can't be negative", + "max-transport-data-points": "Transport data points maximum number ", + "max-transport-data-points-required": "Transport data points maximum number is required.", + "max-transport-data-points-range": "Transport data points maximum number can't be negative", + "max-r-e-executions": "Rule Engine executions maximum number", + "max-r-e-executions-required": "Rule Engine executions maximum number is required.", + "max-r-e-executions-range": "Rule Engine executions maximum number can't be negative", + "max-j-s-executions": "JavaScript executions maximum number ", + "max-j-s-executions-required": "JavaScript executions maximum number is required.", + "max-j-s-executions-range": "JavaScript executions maximum number can't be negative", + "max-d-p-storage-days": "Data points storage days maximum number", + "max-d-p-storage-days-required": "Data points storage days maximum number is required.", + "max-d-p-storage-days-range": "Data points storage days maximum number can't be negative", + "default-storage-ttl-days": "Storage TTL days by default", + "default-storage-ttl-days-required": "Storage TTL days by default is required.", + "default-storage-ttl-days-range": "Storage TTL days by default can't be negative", + "alarms-ttl-days": "Alarms TTL days", "alarms-ttl-days-required": "Alarms TTL days required", "alarms-ttl-days-days-range": "Alarms TTL days can't be negative", - "rpc-ttl-days": "RPC TTL days (0 - unlimited)", + "rpc-ttl-days": "RPC TTL days", "rpc-ttl-days-required": "RPC TTL days required", "rpc-ttl-days-days-range": "RPC TTL days can't be negative", - "max-rule-node-executions-per-message": "Maximum number of rule node executions per message (0 - unlimited)", - "max-rule-node-executions-per-message-required": "Maximum number of rule node executions per message is required.", - "max-rule-node-executions-per-message-range": "Maximum number of rule node executions per message can't be negative", - "max-emails": "Maximum number of emails sent (0 - unlimited)", - "max-emails-required": "Maximum number of emails sent is required.", - "max-emails-range": "Maximum number of emails sent can't be negative", - "max-sms": "Maximum number of SMS sent (0 - unlimited)", - "max-sms-required": "Maximum number of SMS sent is required.", - "max-sms-range": "Maximum number of SMS sent can't be negative", - "max-created-alarms": "Maximum number of alarms created (0 - unlimited)", - "max-created-alarms-required": "Maximum number of alarms created is required.", - "max-created-alarms-range": "Maximum number of alarms created can't be negative", + "max-rule-node-executions-per-message": "Rule node per message executions maximum number", + "max-rule-node-executions-per-message-required": "MRule node per message executions maximum number is required.", + "max-rule-node-executions-per-message-range": "Rule node per message executions maximum number can't be negative", + "max-emails": "Emails sent maximum number", + "max-emails-required": "Emails sent maximum number is required.", + "max-emails-range": "Emails sent maximum number can't be negative", + "max-sms": "SMS sent maximum number", + "max-sms-required": "SMS sent maximum number is required.", + "max-sms-range": "SMS sent maximum number can't be negative", + "max-created-alarms": "Alarms created maximum number", + "max-created-alarms-required": "Alarms created maximum number is required.", + "max-created-alarms-range": "Alarms created maximum number be negative", "no-queue": "No Queue configured", "add-queue": "Add Queue", "queues-with-count": "Queues ({{count}})", - "tenant-rest-limits": "Rate limit for REST requests for tenant", - "customer-rest-limits": "Rate limit for REST requests for customer", + "tenant-rest-limits": "REST requests for tenant", + "customer-rest-limits": "REST requests for customer", "incorrect-pattern-for-rate-limits": "The format is comma separated pairs of capacity and period (in seconds) with a colon between, e.g. 100:1,2000:60", "too-small-value-zero": "The value must be bigger than 0", "too-small-value-one": "The value must be bigger than 1", - "cassandra-tenant-limits-configuration": "Cassandra query rate limit for tenant", - "ws-limit-max-sessions-per-tenant": "Maximum number of WS sessions per tenant", - "ws-limit-max-sessions-per-customer": "Maximum number of WS sessions per customer", - "ws-limit-max-sessions-per-regular-user": "Maximum number of WS sessions per regular user", - "ws-limit-max-sessions-per-public-user": "Maximum number of WS sessions per public user", - "ws-limit-queue-per-session": "Maximum size of WS message queue per session", - "ws-limit-max-subscriptions-per-tenant": "Maximum number of WS subscriptions per tenant", - "ws-limit-max-subscriptions-per-customer": "Maximum number of WS subscriptions per customer", - "ws-limit-max-subscriptions-per-regular-user": "Maximum number of WS subscriptions per regular user", - "ws-limit-max-subscriptions-per-public-user": "Maximum number of WS subscriptions per public user", - "ws-limit-updates-per-session": "Rate limit for WS updates per session" + "cassandra-tenant-limits-configuration": "Cassandra query for tenant", + "ws-limit-max-sessions-per-tenant": "Sessions per tenant maximum number", + "ws-limit-max-sessions-per-customer": "Sessions per customer maximum number", + "ws-limit-max-sessions-per-regular-user": "Sessions per regular user maximum number", + "ws-limit-max-sessions-per-public-user": "Sessions per public user maximum number", + "ws-limit-queue-per-session": "Message queue per session maximum size", + "ws-limit-max-subscriptions-per-tenant": "Subscriptions per tenant maximum number", + "ws-limit-max-subscriptions-per-customer": "Subscriptions per customer maximum number", + "ws-limit-max-subscriptions-per-regular-user": "Subscriptions per regular user maximum number", + "ws-limit-max-subscriptions-per-public-user": "Subscriptions per public user maximum number", + "ws-limit-updates-per-session": "WS updates per session" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", From d903d0ddc0db3a6797552145a5aca4f4bed7e43e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 12:39:50 +0300 Subject: [PATCH 48/80] default Redis cluster password changed, set REDISCLI_AUTH to simplify redis-cli usage with docker exec --- docker/cache-redis-cluster.env | 2 +- docker/docker-compose.redis-cluster.yml | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env index 64058ab556..a3b516063b 100644 --- a/docker/cache-redis-cluster.env +++ b/docker/cache-redis-cluster.env @@ -2,4 +2,4 @@ CACHE_TYPE=redis REDIS_CONNECTION_TYPE=cluster REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 REDIS_USE_DEFAULT_POOL_CONFIG=false -REDIS_PASSWORD=bitnami +REDIS_PASSWORD=thingsboard diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index 17ca4ed13a..0a11b765e8 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -23,7 +23,8 @@ services: volumes: - ./tb-node/redis-cluster-data-0:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-1: @@ -31,7 +32,8 @@ services: volumes: - ./tb-node/redis-cluster-data-1:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-2: @@ -39,7 +41,8 @@ services: volumes: - ./tb-node/redis-cluster-data-2:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-3: @@ -47,7 +50,8 @@ services: volumes: - ./tb-node/redis-cluster-data-3:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-4: @@ -55,7 +59,7 @@ services: volumes: - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-5: @@ -69,9 +73,9 @@ services: - redis-node-3 - redis-node-4 environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' - - 'REDISCLI_AUTH=bitnami' - 'REDIS_CLUSTER_REPLICAS=1' - 'REDIS_CLUSTER_CREATOR=yes' From 7b8d39f66190a8abe91423935f164e846ed90826 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 23 Jun 2022 12:42:03 +0300 Subject: [PATCH 49/80] Undo e.setExternalId(null) in cleanupForComparison --- .../service/sync/ie/importing/impl/BaseEntityImportService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 285401e3bd..26c5a869f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -160,7 +160,6 @@ public abstract class BaseEntityImportService Date: Thu, 23 Jun 2022 12:42:17 +0300 Subject: [PATCH 50/80] REDISCLI_AUTH password --- docker/docker-compose.redis-cluster.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index 0a11b765e8..55c1c3f1db 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -60,6 +60,7 @@ services: - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-5: From 39064c02177aabaebe15626fa3e1bcdcdc287988 Mon Sep 17 00:00:00 2001 From: fe-dev Date: Thu, 23 Jun 2022 13:15:06 +0300 Subject: [PATCH 51/80] UI: refactoring --- .../profile/tenant-profile.component.html | 2 +- ...enant-profile-configuration.component.html | 24 ++++++++++++++----- .../assets/locale/locale.constant-en_US.json | 1 + 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html index e34bb8cad3..21b95cff8b 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.html @@ -47,7 +47,7 @@
-
+
tenant-profile.name diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 770f60067d..0e1553806a 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -17,7 +17,9 @@ -->
- {{ 'tenant-profile.entities' | translate }} (0 - unlimited) + + {{ 'tenant-profile.entities' | translate }} tenant-profile.unlimited +
tenant-profile.maximum-devices @@ -108,7 +110,9 @@
- {{ 'tenant-profile.rule-engine' | translate }} (0 - unlimited) + + {{ 'tenant-profile.rule-engine' | translate }} tenant-profile.unlimited +
tenant-profile.max-r-e-executions @@ -188,7 +192,9 @@
- {{ 'tenant-profile.time-to-live' | translate }} (0 - unlimited) + + {{ 'tenant-profile.time-to-live' | translate }} tenant-profile.unlimited +
tenant-profile.max-d-p-storage-days @@ -244,7 +250,9 @@
- {{ 'tenant-profile.alarms-and-notifications' | translate }} (0 - unlimited) + + {{ 'tenant-profile.alarms-and-notifications' | translate }} tenant-profile.unlimited +
tenant-profile.max-emails @@ -289,7 +297,9 @@
- {{ 'tenant-profile.ota-files-in-bytes' | translate }} (0 - unlimited) + + {{ 'tenant-profile.ota-files-in-bytes' | translate }} tenant-profile.unlimited +
tenant-profile.maximum-resources-sum-data-size @@ -319,7 +329,9 @@
- {{ 'tenant-profile.ws-title' | translate }} (0 - unlimited) + + {{ 'tenant-profile.ws-title' | translate }} tenant-profile.unlimited +
tenant-profile.ws-limit-max-sessions-per-tenant diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cc7b296495..a0c0f00858 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3097,6 +3097,7 @@ "alarms-and-notifications": "Alarms and notifications", "ota-files-in-bytes": "OTA files in bytes", "ws-title": "WS", + "unlimited": "(0 - unlimited)", "maximum-devices": "Devices maximum number", "maximum-devices-required": "Devices maximum number is required.", "maximum-devices-range": "Devices maximum number can't be negative", From 4f8806dd8bc98995f7c1c97d0a415486d20fa5ec Mon Sep 17 00:00:00 2001 From: van-vanich Date: Thu, 23 Jun 2022 13:47:28 +0300 Subject: [PATCH 52/80] fix tests in nosql --- .../timeseries/BaseTimeseriesServiceTest.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 04c3425169..b38afe77f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -230,92 +230,92 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); - for (long i = TS; i <= TS + 100L; i += 10L) { + for (long i = TS; i <= TS + 100000L; i += 10000L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS + 100L + 1L); + saveEntries(deviceId, TS + 100000L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); - for (long i = TS; i <= TS + 80L; i += 10L) { + for (long i = TS; i <= TS + 80000L; i += 10000L) { saveEntries(deviceId, i); } - saveEntries(deviceId, TS + 80L + 1L); + saveEntries(deviceId, TS + 80000L + 1L); - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10) { + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 75, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); } @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); - for (long i = TS - 1L; i <= TS + 100L + 1L; i += 10L) { + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) { saveEntries(deviceId, i); } - List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80, 50, 1, Aggregation.COUNT, DESC_ORDER)); + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(2, entries.size()); - Assert.assertEquals(toTsEntry(TS + 25, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); - Assert.assertEquals(toTsEntry(TS + 65, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); } @Test From 352abb3f608822a704ec58299691efb9532eb764 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Jun 2022 13:52:27 +0300 Subject: [PATCH 53/80] JS-executor: Fixed incorrect parse error code for message --- msa/js-executor/api/jsInvokeMessageProcessor.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index f0ed4c274c..86f441a5b3 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -144,7 +144,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, r }, (err) => { var errorCode; - if (err.message.includes('Script execution timed out')) { + if (err && isString(err.message) && err.message.includes('Script execution timed out')) { errorCode = TIMEOUT_ERROR; } else { errorCode = RUNTIME_ERROR; @@ -321,4 +321,8 @@ function deleteMinUsedScript() { this.scriptMap.delete(prevScriptId); } +function isString(value) { + return typeof value === 'string'; +} + module.exports = JsInvokeMessageProcessor; From 472357109dce21ed3aabcc30b747acfb962339e0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 23 Jun 2022 14:12:03 +0300 Subject: [PATCH 54/80] VC: Detect default remote branch. Fix Entities Version Control REST API mappings to handle branch parameter. --- .../EntitiesVersionControlController.java | 68 +++++++++---------- .../DefaultEntitiesVersionControlService.java | 5 +- .../DefaultGitVersionControlQueueService.java | 9 ++- .../vc/EntitiesVersionControlService.java | 11 ++- .../vc/GitVersionControlQueueService.java | 7 +- .../sync/vc/data/ListBranchesGitRequest.java | 3 +- common/cluster-api/src/main/proto/queue.proto | 7 +- .../common/data/sync/vc/BranchInfo.java | 39 +++++++++++ .../DefaultClusterVersionControlService.java | 6 +- .../sync/vc/DefaultGitRepositoryService.java | 7 +- .../server/service/sync/vc/GitRepository.java | 31 +++++++-- .../service/sync/vc/GitRepositoryService.java | 5 +- .../http/entities-version-control.service.ts | 12 ++-- 13 files changed, 142 insertions(+), 68 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 1bcc304548..fff3c6a4bb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import lombok.Data; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; @@ -39,6 +38,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -56,6 +56,7 @@ import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; @@ -138,10 +139,10 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profile 1 version 1.0\"\n" + " }\n" + "]\n```") - @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) - public DeferredResult> listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, + @GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"}) + public DeferredResult> listEntityVersions(@PathVariable EntityType entityType, @PathVariable UUID externalEntityUuid, + @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -165,9 +166,9 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profiles from dev\"\n" + " }\n" + "]\n```") - @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) - public DeferredResult> listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType, + @GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"}) + public DeferredResult> listEntityTypeVersions(@PathVariable EntityType entityType, + @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -198,8 +199,8 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Devices added\"\n" + " }\n" + "]\n```") - @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) - public DeferredResult> listVersions(@PathVariable String branch, + @GetMapping(value = "/version", params = {"branch", "pageSize", "page"}) + public DeferredResult> listVersions(@RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -216,17 +217,17 @@ public class EntitiesVersionControlController extends BaseController { } - @GetMapping("/entity/{branch}/{entityType}/{versionId}") - public DeferredResult> listEntitiesAtVersion(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable String versionId) throws Exception { + @GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"}) + public DeferredResult> listEntitiesAtVersion(@PathVariable EntityType entityType, + @PathVariable String versionId, + @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); } - @GetMapping("/entity/{branch}/{versionId}") - public DeferredResult> listAllEntitiesAtVersion(@PathVariable String branch, - @PathVariable String versionId) throws Exception { + @GetMapping(value = "/entity/{versionId}", params = {"branch"}) + public DeferredResult> listAllEntitiesAtVersion(@PathVariable String versionId, + @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); } @@ -240,10 +241,10 @@ public class EntitiesVersionControlController extends BaseController { return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); } - @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") - public DeferredResult compareEntityDataToVersion(@PathVariable String branch, - @PathVariable EntityType entityType, + @GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"}) + public DeferredResult compareEntityDataToVersion(@PathVariable EntityType entityType, @PathVariable UUID internalEntityUuid, + @RequestParam String branch, @RequestParam String versionId) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); @@ -318,20 +319,21 @@ public class EntitiesVersionControlController extends BaseController { try { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); final TenantId tenantId = getTenantId(); - ListenableFuture> branches = versionControlService.listBranches(tenantId); + ListenableFuture> branches = versionControlService.listBranches(tenantId); return wrapFuture(Futures.transform(branches, remoteBranches -> { List infos = new ArrayList<>(); - - String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); - if (StringUtils.isNotEmpty(defaultBranch)) { - infos.add(new BranchInfo(defaultBranch, true)); + BranchInfo defaultBranch; + String defaultBranchName = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranchName)) { + defaultBranch = new BranchInfo(defaultBranchName, true); + } else { + defaultBranch = remoteBranches.stream().filter(BranchInfo::isDefault).findFirst().orElse(null); } - - remoteBranches.forEach(branch -> { - if (!branch.equals(defaultBranch)) { - infos.add(new BranchInfo(branch, false)); - } - }); + if (defaultBranch != null) { + infos.add(defaultBranch); + } + infos.addAll(remoteBranches.stream().filter(b -> !b.equals(defaultBranch)) + .map(b -> new BranchInfo(b.getName(), false)).collect(Collectors.toList())); return infos; }, MoreExecutors.directExecutor())); } catch (Exception e) { @@ -339,10 +341,4 @@ public class EntitiesVersionControlController extends BaseController { } } - @Data - public static class BranchInfo { - private final String name; - private final boolean isDefault; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 6ea8115e53..3b625fc086 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,12 +25,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; -import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.cache.CaffeineTbTransactionalCache; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -49,6 +47,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityLoadError; @@ -479,7 +478,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + public ListenableFuture> listBranches(TenantId tenantId) throws Exception { return gitServiceQueue.listBranches(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 99246d282f..628605824e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; @@ -240,7 +241,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture> listBranches(TenantId tenantId) { + public ListenableFuture> listBranches(TenantId tenantId) { ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); } @@ -382,7 +383,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu ((CommitGitRequest) request).getFuture().set(commitResult); } else if (vcResponseMsg.hasListBranchesResponse()) { var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); - ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList()); + ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList().stream().map(this::getBranchInfo).collect(Collectors.toList())); } else if (vcResponseMsg.hasListEntitiesResponse()) { var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); ((ListEntitiesGitRequest) request).getFuture().set( @@ -439,6 +440,10 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); } + private BranchInfo getBranchInfo(TransportProtos.BranchInfoProto proto) { + return new BranchInfo(proto.getName(), proto.getIsDefault()); + } + @SuppressWarnings("rawtypes") @SneakyThrows private EntityExportData toData(String data) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 3ae68e33d0..5aa4c40008 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,22 +16,21 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; -import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; -import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -60,7 +59,7 @@ public interface EntitiesVersionControlService { ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; - ListenableFuture> listBranches(TenantId tenantId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; RepositorySettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index a1aad04701..a1e64da8f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -24,14 +24,15 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; -import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import java.util.List; @@ -55,7 +56,7 @@ public interface GitVersionControlQueueService { ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); - ListenableFuture> listBranches(TenantId tenantId); + ListenableFuture> listBranches(TenantId tenantId); ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java index c045030dc7..4d89efa14d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java @@ -16,10 +16,11 @@ package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import java.util.List; -public class ListBranchesGitRequest extends PendingGitRequest> { +public class ListBranchesGitRequest extends PendingGitRequest> { public ListBranchesGitRequest(TenantId tenantId) { super(tenantId); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 3079978bce..b7362c7f03 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -765,8 +765,13 @@ message ListEntitiesResponseMsg { message ListBranchesRequestMsg { } +message BranchInfoProto { + string name = 1; + bool isDefault = 2; +} + message ListBranchesResponseMsg { - repeated string branches = 1; + repeated BranchInfoProto branches = 1; } message EntityContentRequestMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java new file mode 100644 index 0000000000..fcf1f4470e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.Data; + +import java.util.Objects; + +@Data +public class BranchInfo { + private final String name; + private final boolean isDefault; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BranchInfo that = (BranchInfo) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index b28618372e..f74f18ddaa 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AddMsg; +import org.thingsboard.server.gen.transport.TransportProtos.BranchInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg; @@ -332,7 +333,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { - var branches = vcService.listBranches(ctx.getTenantId()); + var branches = vcService.listBranches(ctx.getTenantId()).stream() + .map(branchInfo -> BranchInfoProto.newBuilder() + .setName(branchInfo.getName()) + .setIsDefault(branchInfo.isDefault()).build()).collect(Collectors.toList()); reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 99dbebc3c9..d3615f7228 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -29,8 +29,9 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.GitRepository.Diff; @@ -84,7 +85,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); repository.resetAndClean(); - if (repository.listRemoteBranches().contains(branch)) { + if (repository.listRemoteBranches().contains(new BranchInfo(branch, false))) { repository.merge(branch); } } catch (IOException | GitAPIException gitAPIException) { @@ -182,7 +183,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listBranches(TenantId tenantId) { + public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); try { return repository.listRemoteBranches(); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 2b2802d003..392c686a14 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -41,10 +41,12 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; @@ -58,6 +60,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; @@ -69,7 +72,12 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.PublicKey; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -84,6 +92,8 @@ public class GitRepository { @Getter private final String directory; + private ObjectId headId; + private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.settings = settings; @@ -135,8 +145,12 @@ public class GitRepository { } public void fetch() throws GitAPIException { - execute(git.fetch() + FetchResult result = execute(git.fetch() .setRemoveDeletedRefs(true)); + Ref head = result.getAdvertisedRef(Constants.HEAD); + if (head != null) { + this.headId = head.getObjectId(); + } } public void deleteLocalBranchIfExists(String branch) throws GitAPIException { @@ -162,13 +176,11 @@ public class GitRepository { .include(branchId)); } - - public List listRemoteBranches() throws GitAPIException { + public List listRemoteBranches() throws GitAPIException { return execute(git.branchList() .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .filter(ref -> !ref.getName().equals(Constants.HEAD)) - .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) - .map(name -> StringUtils.removeStart(name, "origin/")) + .map(this::toBranchInfo) .distinct().collect(Collectors.toList()); } @@ -325,6 +337,13 @@ public class GitRepository { .collect(Collectors.toList()); } + private BranchInfo toBranchInfo(Ref ref) { + String name = org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()); + String branchName = StringUtils.removeStart(name, "origin/"); + boolean isDefault = this.headId != null && this.headId.equals(ref.getObjectId()); + return new BranchInfo(branchName, isDefault); + } + private Commit toCommit(RevCommit revCommit) { return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 057fa9c1ab..d4f909df9e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -19,8 +19,9 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.GitRepository.Diff; @@ -57,7 +58,7 @@ public interface GitRepositoryService { void abort(PendingCommit commit); - List listBranches(TenantId tenantId); + List listBranches(TenantId tenantId); String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index ec0b6ba2ae..d7e892f1db 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -118,20 +118,23 @@ export class EntitiesVersionControlService { public listEntityVersions(pageLink: PageLink, branch: string, externalEntityId: EntityId, config?: RequestConfig): Observable> { - return this.http.get>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}&branch=${encodedBranch}`, defaultHttpOptionsFromConfig(config)); } public listEntityTypeVersions(pageLink: PageLink, branch: string, entityType: EntityType, config?: RequestConfig): Observable> { - return this.http.get>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version/${entityType}${pageLink.toQuery()}&branch=${encodedBranch}`, defaultHttpOptionsFromConfig(config)); } public listVersions(pageLink: PageLink, branch: string, config?: RequestConfig): Observable> { - return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version${pageLink.toQuery()}&branch=${encodedBranch}`, defaultHttpOptionsFromConfig(config)); } @@ -160,7 +163,8 @@ export class EntitiesVersionControlService { entityId: EntityId, versionId: string, config?: RequestConfig): Observable { - return this.http.get(`/api/entities/vc/diff/${branch}/${entityId.entityType}/${entityId.id}?versionId=${versionId}`, + const encodedBranch = encodeURIComponent(branch); + return this.http.get(`/api/entities/vc/diff/${entityId.entityType}/${entityId.id}?branch=${encodedBranch}&versionId=${versionId}`, defaultHttpOptionsFromConfig(config)); } From 99de983635a4b5f49d3de77832d31352deabad02 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 15:04:03 +0300 Subject: [PATCH 55/80] docker-compose: bump zookeeper version from 3.5 up to 3.8.0, disable new admin feature (ZOO_ADMINSERVER_ENABLED=false) --- docker/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 62969b1051..3e0f21391d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,12 +20,13 @@ version: '2.2' services: zookeeper: restart: always - image: "zookeeper:3.5" + image: "zookeeper:3.8.0" ports: - "2181" environment: ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 + ZOO_ADMINSERVER_ENABLED: "false" tb-js-executor: restart: always image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" From 46682b8bb6cb166e30e6226ae54bbcbc003c3368 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 15:05:41 +0300 Subject: [PATCH 56/80] docker-compose: cleanup depends_on for external services --- docker/docker-compose.aws-sqs.yml | 26 -------------------------- docker/docker-compose.confluent.yml | 8 -------- 2 files changed, 34 deletions(-) diff --git a/docker/docker-compose.aws-sqs.yml b/docker/docker-compose.aws-sqs.yml index e50d0d424f..dff3f595ad 100644 --- a/docker/docker-compose.aws-sqs.yml +++ b/docker/docker-compose.aws-sqs.yml @@ -23,59 +23,33 @@ services: tb-core1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index 3d5abd0abe..077acdca98 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -23,23 +23,15 @@ services: tb-core1: env_file: - queue-confluent.env - depends_on: - - redis tb-core2: env_file: - queue-confluent.env - depends_on: - - redis tb-rule-engine1: env_file: - queue-confluent.env - depends_on: - - redis tb-rule-engine2: env_file: - queue-confluent.env - depends_on: - - redis tb-mqtt-transport1: env_file: - queue-confluent.env From dd2d1e6443751c2ef8c407ee092998bd438135f0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jun 2022 15:38:31 +0300 Subject: [PATCH 57/80] Removed redundant variable --- .../dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index d68271798a..69f16d4bbc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -131,8 +131,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq long startTs = startPeriod; long endTs = Math.min(startPeriod + step, endPeriod + 1); long ts = startTs + (endTs - startTs) / 2; - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); - ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, subQuery.getKey(), startTs, endTs, ts, query.getAggregation()); + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); futures.add(aggregateTsKvEntry); startPeriod = endTs; } From 56e068fe07e1629c365ba949426c0b57fb2dbd03 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 15:47:22 +0300 Subject: [PATCH 58/80] docker-compose: fixed tb-vc-executor for queue-aws-sqs --- docker/docker-compose.aws-sqs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/docker-compose.aws-sqs.yml b/docker/docker-compose.aws-sqs.yml index dff3f595ad..331902851f 100644 --- a/docker/docker-compose.aws-sqs.yml +++ b/docker/docker-compose.aws-sqs.yml @@ -53,3 +53,9 @@ services: tb-snmp-transport: env_file: - queue-aws-sqs.env + tb-vc-executor1: + env_file: + - queue-aws-sqs.env + tb-vc-executor2: + env_file: + - queue-aws-sqs.env From 8a9ce8d27216dcabc06b86f12fecfbd98e91e8b7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 15:56:47 +0300 Subject: [PATCH 59/80] docker-compose: cleanup zookeeper dependency from hybrid. The necessary dependency in the main docker-compose.yml along with zookeeper service --- docker/docker-compose.hybrid.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index e3ba32b779..197e1f1dcb 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -38,27 +38,23 @@ services: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-core2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-rule-engine1: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-rule-engine2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra From 64a82991b5b02ba590b0a6b6ea978da65ef809a4 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 23 Jun 2022 16:24:09 +0300 Subject: [PATCH 60/80] Fix rule chain comparing on import --- .../service/sync/ie/exporting/impl/BaseEntityExportService.java | 2 -- .../service/sync/ie/importing/impl/RuleChainImportService.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 169b7f273c..01626368dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -41,8 +41,6 @@ public abstract class BaseEntityExportService(); } - ; - public abstract Set getSupportedEntityTypes(); protected void replaceUuidsRecursively(EntitiesExportCtx ctx, JsonNode node, Set skipFieldsSet) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index dadea35e68..c66f4a6b08 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -119,7 +119,7 @@ public class RuleChainImportService extends BaseEntityImportService Date: Thu, 23 Jun 2022 17:00:55 +0300 Subject: [PATCH 61/80] Always update widgets on import --- .../sync/ie/importing/impl/WidgetsBundleImportService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java index 550f00952a..f76b3cb9c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java @@ -86,6 +86,11 @@ public class WidgetsBundleImportService extends BaseEntityImportService Date: Thu, 23 Jun 2022 17:27:12 +0300 Subject: [PATCH 62/80] Fix cluster issues --- .../edge/rpc/sync/DefaultEdgeRequestsService.java | 2 ++ .../service/entitiy/AbstractTbEntityService.java | 15 +++++++++++++-- .../entitiy/asset/DefaultTbAssetService.java | 2 +- .../customer/DefaultTbCustomerService.java | 2 +- .../dashboard/DefaultTbDashboardService.java | 2 +- .../entitiy/device/DefaultTbDeviceService.java | 2 +- .../DefaultTbDeviceProfileService.java | 2 +- .../install/DefaultSystemDataLoaderService.java | 2 ++ .../install/SqlDatabaseUpgradeService.java | 2 ++ .../install/update/DefaultDataUpdateService.java | 2 ++ .../dao/device/DeviceProfileServiceImpl.java | 2 ++ .../server/dao/queue/BaseQueueService.java | 2 ++ .../validator/DeviceProfileDataValidator.java | 2 +- .../dao/sqlts/BaseAbstractSqlTimeseriesDao.java | 3 ++- .../server/dao/tenant/TenantServiceImpl.java | 1 + 15 files changed, 34 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index 90dbaf3312..210ad30c44 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; @@ -101,6 +102,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { @Autowired private DeviceService deviceService; + @Lazy @Autowired private TbEntityViewService entityViewService; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 7abe031f2e..e3d91b7a7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -65,6 +65,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.ota.OtaPackageStateService; @@ -78,6 +79,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; @Slf4j @@ -128,8 +130,8 @@ public abstract class AbstractTbEntityService { @Autowired protected DashboardService dashboardService; - @Autowired - protected EntitiesVersionControlService vcService; + @Autowired(required = false) + private EntitiesVersionControlService vcService; @Autowired protected EntityViewService entityViewService; @Lazy @@ -249,4 +251,13 @@ public abstract class AbstractTbEntityService { } return result; } + + protected ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception { + if (vcService != null) { + return vcService.autoCommit(user, entityId); + } else { + // We do not support auto-commit for rule engine + return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!")); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index e118346588..e71c41b2f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -47,7 +47,7 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb TenantId tenantId = asset.getTenantId(); try { Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - vcService.autoCommit(user, savedAsset.getId()); + autoCommit(user, savedAsset.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), savedAsset, savedAsset.getCustomerId(), actionType, user); return savedAsset; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java index 536e1be976..4d5a85d811 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java @@ -42,7 +42,7 @@ public class DefaultTbCustomerService extends AbstractTbEntityService implements TenantId tenantId = customer.getTenantId(); try { Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - vcService.autoCommit(user, savedCustomer.getId()); + autoCommit(user, savedCustomer.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, null, actionType, user); return savedCustomer; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java index a475ff77da..c7a04fbb78 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java @@ -48,7 +48,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement TenantId tenantId = dashboard.getTenantId(); try { Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - vcService.autoCommit(user, savedDashboard.getId()); + autoCommit(user, savedDashboard.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard, null, actionType, user); return savedDashboard; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index 5199887940..5c232e4712 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -54,7 +54,7 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T ActionType actionType = device.getId() == null ? ActionType.ADDED : ActionType.UPDATED; try { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - vcService.autoCommit(user, savedDevice.getId()); + autoCommit(user, savedDevice.getId()); notificationEntityService.notifyCreateOrUpdateDevice(tenantId, savedDevice.getId(), savedDevice.getCustomerId(), savedDevice, oldDevice, actionType, user); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java index 3a0c1f1a23..a92e5109f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java @@ -54,7 +54,7 @@ public class DefaultTbDeviceProfileService extends AbstractTbEntityService imple } } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - vcService.autoCommit(user, savedDeviceProfile.getId()); + autoCommit(user, savedDeviceProfile.getId()); tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedDeviceProfile.getId(), actionType.equals(ActionType.ADDED) ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 50fcb28387..14e49e056f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; @@ -162,6 +163,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Getter private boolean persistActivityToTelemetry; + @Lazy @Autowired private QueueService queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 306102c66c..e428d55d91 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntitySubtype; @@ -121,6 +122,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @Autowired private ApiUsageStateService apiUsageStateService; + @Lazy @Autowired private QueueService queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index c6576097ee..9bfa86b3ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -117,6 +118,7 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private TenantProfileService tenantProfileService; + @Lazy @Autowired private QueueService queueService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 95f2f034fe..a6cfa33fb8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.Device; @@ -69,6 +70,7 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService deviceProfileValidator; + @Lazy @Autowired private QueueService queueService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java index a0254a4923..45b5b2cc7e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.QueueId; @@ -43,6 +44,7 @@ public class BaseQueueService extends AbstractEntityService implements QueueServ @Autowired private QueueDao queueDao; + @Lazy @Autowired private TbTenantProfileCache tenantProfileCache; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index 429a328c53..b1fb9b150b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -90,8 +90,8 @@ public class DeviceProfileDataValidator extends DataValidator { private DeviceDao deviceDao; @Autowired private TenantService tenantService; - @Autowired @Lazy + @Autowired private QueueService queueService; @Autowired private OtaPackageService otaPackageService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index dd9d15b8b1..82e54168bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; @@ -59,7 +60,7 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); - } catch (ConstraintViolationException e) { + } catch (DataIntegrityViolationException | ConstraintViolationException e) { tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 35a22f8281..a6fefd3ead 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -107,6 +107,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService tenantValidator; + @Lazy @Autowired private QueueService queueService; From 9cd101fd67a111675f32c6ed3bc333007a169a76 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 23 Jun 2022 17:28:26 +0300 Subject: [PATCH 63/80] Not update defaultQueueId of device profile on import --- .../sync/ie/importing/impl/DeviceProfileImportService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java index 26bde001f5..1e26940be4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -52,6 +52,7 @@ public class DeviceProfileImportService extends BaseEntityImportService Date: Thu, 23 Jun 2022 18:34:38 +0300 Subject: [PATCH 64/80] Fix comparing of dashboard to existing one on import --- .../sync/ie/importing/impl/DashboardImportService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 2534ffc1b2..f5e550b433 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -117,6 +117,11 @@ public class DashboardImportService extends BaseEntityImportService exportData, Dashboard prepared, Dashboard existing) { + return super.compare(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration()); + } + @Override protected void onEntitySaved(SecurityUser user, Dashboard savedDashboard, Dashboard oldDashboard) throws ThingsboardException { super.onEntitySaved(user, savedDashboard, oldDashboard); From 011caa20fd3954378c13d2b5c74a1a4150ffc1d0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 23 Jun 2022 18:58:29 +0300 Subject: [PATCH 65/80] Add hybrid mode for black-box tests. Update cassandra version in docker compose. Fix NPE in AbstractBufferedRateExecutor - ignore profile rate limits for SYS_TENANT_ID. --- .../util/AbstractBufferedRateExecutor.java | 9 +++- docker/docker-compose.cassandra.volumes.yml | 27 ++++++++++++ docker/docker-compose.hybrid.yml | 2 +- msa/black-box-tests/README.md | 4 ++ .../server/msa/ContainerTestSuite.java | 23 ++++++++--- .../server/msa/ThingsBoardDbInstaller.java | 41 ++++++++++++++----- 6 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 docker/docker-compose.cassandra.volumes.yml diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java index 2850349470..4c45239921 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java @@ -116,8 +116,13 @@ public abstract class AbstractBufferedRateExecutor> extends DockerComposeContainer { - public DockerComposeContainerImpl(File... composeFiles) { + public DockerComposeContainerImpl(List composeFiles) { super(composeFiles); } @@ -72,10 +77,11 @@ public class ContainerTestSuite { } } - testContainer = new DockerComposeContainerImpl<>( - new File(targetDir + "docker-compose.yml"), + List composeFiles = new ArrayList<>(Arrays.asList(new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), - new File(targetDir + "docker-compose.postgres.yml"), + IS_HYBRID_MODE + ? new File(targetDir + "docker-compose.hybrid.yml") + : new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER @@ -83,8 +89,13 @@ public class ContainerTestSuite { : new File("./../../docker/docker-compose.redis.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") - : new File("./../../docker/docker-compose.redis.volumes.yml") - ) + : new File("./../../docker/docker-compose.redis.volumes.yml"))); + + if (IS_HYBRID_MODE) { + composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); + } + + testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 08a333ab52..08cadbceb2 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -21,10 +21,7 @@ import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import java.io.File; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -32,7 +29,10 @@ import java.util.stream.IntStream; public class ThingsBoardDbInstaller extends ExternalResource { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); + final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; + + private final static String CASSANDRA_DATA_VOLUME = "tb-cassandra-test-data-volume"; private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume"; private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume"; @@ -46,6 +46,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final DockerComposeExecutor dockerCompose; private final String postgresDataVolume; + private final String cassandraDataVolume; private final String redisDataVolume; private final String redisClusterDataVolume; @@ -60,10 +61,13 @@ public class ThingsBoardDbInstaller extends ExternalResource { public ThingsBoardDbInstaller() { log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); - List composeFiles = Arrays.asList( + log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); + List composeFiles = new ArrayList<>(Arrays.asList( new File("./../../docker/docker-compose.yml"), new File("./../../docker/docker-compose.volumes.yml"), - new File("./../../docker/docker-compose.postgres.yml"), + IS_HYBRID_MODE + ? new File("./../../docker/docker-compose.hybrid.yml") + : new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.volumes.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") @@ -71,12 +75,16 @@ public class ThingsBoardDbInstaller extends ExternalResource { IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") : new File("./../../docker/docker-compose.redis.volumes.yml") - ); + )); + if (IS_HYBRID_MODE) { + composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); + } String identifier = Base58.randomString(6).toLowerCase(); String project = identifier + Base58.randomString(6).toLowerCase(); postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; + cassandraDataVolume = project + "_" + CASSANDRA_DATA_VOLUME; redisDataVolume = project + "_" + REDIS_DATA_VOLUME; redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME; @@ -91,6 +99,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { env = new HashMap<>(); env.put("POSTGRES_DATA_VOLUME", postgresDataVolume); + if (IS_HYBRID_MODE) { + env.put("CASSANDRA_DATA_VOLUME", cassandraDataVolume); + } env.put("TB_LOG_VOLUME", tbLogVolume); env.put("TB_COAP_TRANSPORT_LOG_VOLUME", tbCoapTransportLogVolume); env.put("TB_LWM2M_TRANSPORT_LOG_VOLUME", tbLwm2mTransportLogVolume); @@ -119,6 +130,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + postgresDataVolume); dockerCompose.invokeDocker(); + if (IS_HYBRID_MODE) { + dockerCompose.withCommand("volume create " + cassandraDataVolume); + dockerCompose.invokeDocker(); + } + dockerCompose.withCommand("volume create " + tbLogVolume); dockerCompose.invokeDocker(); @@ -140,20 +156,23 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); - String redisService = ""; + String additionalServices = ""; + if (IS_HYBRID_MODE) { + additionalServices += " cassandra"; + } if (IS_REDIS_CLUSTER) { for (int i = 0; i < 6; i++) { - redisService = redisService + " redis-node-" + i; + additionalServices = additionalServices + " redis-node-" + i; dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i); dockerCompose.invokeDocker(); } } else { - redisService = "redis"; + additionalServices += " redis"; dockerCompose.withCommand("volume create " + redisDataVolume); dockerCompose.invokeDocker(); } - dockerCompose.withCommand("up -d postgres " + redisService); + dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); From 0d82903a4d69dd4c42010fafdb02622e105ff702 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 19:49:27 +0300 Subject: [PATCH 66/80] docker-compose: synced with docker-compose project, improved merge experience --- docker/compose-utils.sh | 10 +++++----- docker/docker-compose.kafka.yml | 2 +- docker/docker-compose.pubsub.yml | 1 - docker/docker-compose.rabbitmq.yml | 2 +- docker/docker-create-log-folders.sh | 20 ++++++++++---------- docker/docker-install-tb.sh | 8 ++++++-- docker/docker-remove-services.sh | 4 +++- docker/docker-start-services.sh | 4 +++- docker/docker-stop-services.sh | 4 +++- docker/docker-update-service.sh | 8 ++++++-- docker/docker-upgrade-tb.sh | 18 +++++++++++++----- 11 files changed, 51 insertions(+), 30 deletions(-) diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index d60c81ec6a..28601afa3c 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -26,7 +26,7 @@ function additionalComposeArgs() { ADDITIONAL_COMPOSE_ARGS="-f docker-compose.hybrid.yml" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_ARGS @@ -55,7 +55,7 @@ function additionalComposeQueueArgs() { ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml" ;; *) - echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or confluent or aws-sqs or pubsub or rabbitmq or service-bus." >&2 + echo "Unknown Queue service TB_QUEUE_TYPE value specified in the .env file: '${TB_QUEUE_TYPE}'. Should be either 'kafka' or 'confluent' or 'aws-sqs' or 'pubsub' or 'rabbitmq' or 'service-bus'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_QUEUE_ARGS @@ -85,7 +85,7 @@ function additionalComposeCacheArgs() { CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac echo $CACHE_COMPOSE_ARGS @@ -102,7 +102,7 @@ function additionalStartupServices() { ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac @@ -115,7 +115,7 @@ function additionalStartupServices() { ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 82fbb6ed62..e6fa0c489f 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -95,4 +95,4 @@ services: env_file: - queue-kafka.env depends_on: - - kafka \ No newline at end of file + - kafka diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index c03132d730..7c122d0835 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -59,4 +59,3 @@ services: tb-vc-executor2: env_file: - queue-pubsub.env - diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index d1acc32014..1f1cf1554c 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -58,4 +58,4 @@ services: - queue-rabbitmq.env tb-vc-executor2: env_file: - - queue-rabbitmq.env \ No newline at end of file + - queue-rabbitmq.env diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 83257c0e66..8daf11c624 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -15,7 +15,7 @@ # limitations under the License. # -mkdir -p tb-node/log/ && sudo chown -R 799:799 tb-node/log/ +mkdir -p tb-node/log && sudo chown -R 799:799 tb-node/log mkdir -p tb-transports/coap/log && sudo chown -R 799:799 tb-transports/coap/log @@ -29,23 +29,23 @@ mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log -mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/ +mkdir -p tb-node/postgres && sudo chown -R 999:999 tb-node/postgres source .env CACHE="${CACHE:-redis}" case $CACHE in redis) - mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/ + mkdir -p tb-node/redis-data && sudo chown -R 1001:0 tb-node/redis-data ;; redis-cluster) - mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/ - mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/ - mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/ - mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/ - mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/ - mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/ + mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:0 tb-node/redis-cluster-data-0 + mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:0 tb-node/redis-cluster-data-1 + mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:0 tb-node/redis-cluster-data-2 + mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:0 tb-node/redis-cluster-data-3 + mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:0 tb-node/redis-cluster-data-4 + mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:0 tb-node/redis-cluster-data-5 ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac \ No newline at end of file diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 27e84c4d0b..7263fec7fa 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -50,9 +50,13 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES + docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \ + tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 36e464f45d..769150c1f3 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 7993ab425b..9e5d08add9 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index ae8a3c0b81..5b09aea204 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index a02f9e208a..027280635d 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -25,5 +25,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull $@ +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index c20543783b..260e14cf8b 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -48,8 +48,16 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 - -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES - -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull \ + tb-core1 + +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES + +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \ + tb-core1 From f0292849c7bcfd1d52912d7d0b0355df68521caf Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 21:00:16 +0300 Subject: [PATCH 67/80] docker-compose create folder tb-node/cassandra --- docker/docker-create-log-folders.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 8daf11c624..d1d0277d3c 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -31,6 +31,8 @@ mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log mkdir -p tb-node/postgres && sudo chown -R 999:999 tb-node/postgres +mkdir -p tb-node/cassandra && sudo chown -R 999:999 tb-node/cassandra + source .env CACHE="${CACHE:-redis}" case $CACHE in From d4ba0c3f26a6ac3cb7e48d3884c304ee8ee1825d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 21:25:55 +0300 Subject: [PATCH 68/80] docker-compose multiline refactor for better merge --- docker/docker-install-tb.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 7263fec7fa..9129c72796 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -51,7 +51,8 @@ ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then docker-compose \ - -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES fi docker-compose \ From 2aeb735b9569eb15c9fcf853c24b5ac5631011a9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 23:31:57 +0300 Subject: [PATCH 69/80] docker-compose: redis data chown -R 1001:1001 --- docker/docker-create-log-folders.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index d1d0277d3c..6224b7040b 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -37,15 +37,15 @@ source .env CACHE="${CACHE:-redis}" case $CACHE in redis) - mkdir -p tb-node/redis-data && sudo chown -R 1001:0 tb-node/redis-data + mkdir -p tb-node/redis-data && sudo chown -R 1001:1001 tb-node/redis-data ;; redis-cluster) - mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:0 tb-node/redis-cluster-data-0 - mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:0 tb-node/redis-cluster-data-1 - mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:0 tb-node/redis-cluster-data-2 - mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:0 tb-node/redis-cluster-data-3 - mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:0 tb-node/redis-cluster-data-4 - mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:0 tb-node/redis-cluster-data-5 + mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-0 + mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-1 + mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-2 + mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-3 + mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-4 + mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-5 ;; *) echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 From d828ea2e176177fe72ce1570cdc43778d04c12b9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 24 Jun 2022 03:06:53 +0300 Subject: [PATCH 70/80] black-box-test: cleanup code --- .../thingsboard/server/msa/ContainerTestSuite.java | 11 ++++++----- .../server/msa/ThingsBoardDbInstaller.java | 6 +++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 6802f19214..1c4b01c5d1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -77,7 +77,8 @@ public class ContainerTestSuite { } } - List composeFiles = new ArrayList<>(Arrays.asList(new File(targetDir + "docker-compose.yml"), + List composeFiles = new ArrayList<>(Arrays.asList( + new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), IS_HYBRID_MODE ? new File(targetDir + "docker-compose.hybrid.yml") @@ -85,11 +86,11 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml"), + ? new File(targetDir + "docker-compose.redis-cluster.yml") + : new File(targetDir + "docker-compose.redis.yml"), IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") - : new File("./../../docker/docker-compose.redis.volumes.yml"))); + ? new File(targetDir + "docker-compose.redis-cluster.volumes.yml") + : new File(targetDir + "docker-compose.redis.volumes.yml"))); if (IS_HYBRID_MODE) { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 08cadbceb2..b627d2787a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -21,7 +21,11 @@ import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import java.io.File; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; From 13765a4dfc0b8713441b93a7e929b53396a2ea38 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jun 2022 11:42:23 +0300 Subject: [PATCH 71/80] Reduced redundant logging --- application/src/main/conf/logback.xml | 2 + .../AnnotationComponentDiscoveryService.java | 10 ++--- application/src/main/resources/logback.xml | 39 +++++++++++-------- .../src/main/resources/thingsboard.yml | 3 ++ msa/vc-executor/src/main/conf/logback.xml | 2 + .../src/main/resources/logback.xml | 3 ++ transport/coap/src/main/conf/logback.xml | 2 + transport/coap/src/main/resources/logback.xml | 3 ++ transport/http/src/main/conf/logback.xml | 2 + transport/http/src/main/resources/logback.xml | 3 ++ transport/lwm2m/src/main/conf/logback.xml | 2 + .../lwm2m/src/main/resources/logback.xml | 3 ++ transport/mqtt/src/main/conf/logback.xml | 2 + transport/mqtt/src/main/resources/logback.xml | 3 ++ transport/snmp/src/main/conf/logback.xml | 2 + transport/snmp/src/main/resources/logback.xml | 3 ++ 16 files changed, 62 insertions(+), 22 deletions(-) diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 898128c1ff..6d2c95ee84 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java index f279a29aac..7a95bf78f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java @@ -161,20 +161,20 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe configurationDescriptor.set("nodeDefinition", node); scannedComponent.setConfigurationDescriptor(configurationDescriptor); scannedComponent.setClazz(clazzName); - log.info("Processing scanned component: {}", scannedComponent); + log.debug("Processing scanned component: {}", scannedComponent); } catch (Exception e) { log.error("Can't initialize component {}, due to {}", def.getBeanClassName(), e.getMessage(), e); throw new RuntimeException(e); } ComponentDescriptor persistedComponent = componentDescriptorService.findByClazz(TenantId.SYS_TENANT_ID, clazzName); if (persistedComponent == null) { - log.info("Persisting new component: {}", scannedComponent); + log.debug("Persisting new component: {}", scannedComponent); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); } else if (scannedComponent.equals(persistedComponent)) { - log.info("Component is already persisted: {}", persistedComponent); + log.debug("Component is already persisted: {}", persistedComponent); scannedComponent = persistedComponent; } else { - log.info("Component {} will be updated to {}", persistedComponent, scannedComponent); + log.debug("Component {} will be updated to {}", persistedComponent, scannedComponent); componentDescriptorService.deleteByClazz(TenantId.SYS_TENANT_ID, persistedComponent.getClazz()); scannedComponent.setId(persistedComponent.getId()); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); @@ -224,7 +224,7 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe @Override public void discoverComponents() { registerRuleNodeComponents(); - log.info("Found following definitions: {}", components.values()); + log.debug("Found following definitions: {}", components.values()); } @Override diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 995ebb43c4..4784ad2d43 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,26 +25,31 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 07233bb241..f6312a7659 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -428,6 +428,9 @@ cache: timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}" maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}" +#Disable this because it is not required. +spring.data.redis.repositories.enabled: false + redis: # standalone or cluster connection: diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml index d62cf2b3f5..c0f33852a6 100644 --- a/msa/vc-executor/src/main/conf/logback.xml +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -35,6 +35,8 @@ + + diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml index 572d093dde..e25a1995c1 100644 --- a/msa/vc-executor/src/main/resources/logback.xml +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -28,6 +28,9 @@ + + + diff --git a/transport/coap/src/main/conf/logback.xml b/transport/coap/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/coap/src/main/conf/logback.xml +++ b/transport/coap/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/coap/src/main/resources/logback.xml b/transport/coap/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/coap/src/main/resources/logback.xml +++ b/transport/coap/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/http/src/main/conf/logback.xml b/transport/http/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/http/src/main/conf/logback.xml +++ b/transport/http/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/http/src/main/resources/logback.xml b/transport/http/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/http/src/main/resources/logback.xml +++ b/transport/http/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/lwm2m/src/main/conf/logback.xml b/transport/lwm2m/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/lwm2m/src/main/conf/logback.xml +++ b/transport/lwm2m/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/lwm2m/src/main/resources/logback.xml b/transport/lwm2m/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/lwm2m/src/main/resources/logback.xml +++ b/transport/lwm2m/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/mqtt/src/main/conf/logback.xml b/transport/mqtt/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/mqtt/src/main/conf/logback.xml +++ b/transport/mqtt/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/mqtt/src/main/resources/logback.xml b/transport/mqtt/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/mqtt/src/main/resources/logback.xml +++ b/transport/mqtt/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/snmp/src/main/conf/logback.xml b/transport/snmp/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/snmp/src/main/conf/logback.xml +++ b/transport/snmp/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/snmp/src/main/resources/logback.xml b/transport/snmp/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/snmp/src/main/resources/logback.xml +++ b/transport/snmp/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + From 60deed2a73f1596268c37f83e90c26c612c29f5d Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 24 Jun 2022 13:14:51 +0300 Subject: [PATCH 72/80] Use TbNotificationEntityService in VC services --- .../entitiy/DefaultTbNotificationEntityService.java | 2 +- .../service/entitiy/TbNotificationEntityService.java | 2 +- .../entitiy/tenant/DefaultTbTenantService.java | 2 +- .../sync/ie/DefaultEntitiesExportImportService.java | 9 ++++----- .../sync/ie/importing/impl/AssetImportService.java | 11 ----------- .../ie/importing/impl/BaseEntityImportService.java | 5 ++--- .../sync/ie/importing/impl/CustomerImportService.java | 8 -------- .../ie/importing/impl/DashboardImportService.java | 8 -------- .../sync/ie/importing/impl/DeviceImportService.java | 5 +++-- .../ie/importing/impl/DeviceProfileImportService.java | 4 +--- .../ie/importing/impl/EntityViewImportService.java | 6 +++--- .../ie/importing/impl/RuleChainImportService.java | 4 +++- .../ie/importing/impl/WidgetsBundleImportService.java | 1 - 13 files changed, 19 insertions(+), 48 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java index 7ee7696f1a..76477c974b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java @@ -128,7 +128,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS } @Override - public void notifyCreateOruUpdateTenant(Tenant tenant, ComponentLifecycleEvent event) { + public void notifyCreateOrUpdateTenant(Tenant tenant, ComponentLifecycleEvent event) { tbClusterService.onTenantChange(tenant, null); tbClusterService.broadcastEntityStateChangeEvent(tenant.getId(), tenant.getId(), event); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java index c1131761dc..5ce91046d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java @@ -71,7 +71,7 @@ public interface TbNotificationEntityService { E entity, ActionType actionType, SecurityUser user, Object... additionalInfo); - void notifyCreateOruUpdateTenant(Tenant tenant, ComponentLifecycleEvent event); + void notifyCreateOrUpdateTenant(Tenant tenant, ComponentLifecycleEvent event); void notifyDeleteTenant(Tenant tenant); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 22baed8366..d3c12cb3f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -54,7 +54,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T installScripts.createDefaultEdgeRuleChains(savedTenant.getId()); } tenantProfileCache.evict(savedTenant.getId()); - notificationEntityService.notifyCreateOruUpdateTenant(savedTenant, created ? + notificationEntityService.notifyCreateOrUpdateTenant(savedTenant, created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); TenantProfile oldTenantProfile = oldTenant != null ? tenantProfileService.findTenantProfileById(TenantId.SYS_TENANT_ID, oldTenant.getTenantProfileId()) : null; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index b2abe2a581..2b96ebd4ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -34,6 +34,7 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.apiusage.RateLimitService; +import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; @@ -57,9 +58,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final EntityActionService entityActionService; private final RelationService relationService; private final RateLimitService rateLimitService; + private final TbNotificationEntityService entityNotificationService; protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -109,10 +110,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations())); for (EntityRelation relation : ctx.getRelations()) { - entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, - ActionType.RELATION_ADD_OR_UPDATE, null, relation); - entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, - ActionType.RELATION_ADD_OR_UPDATE, null, relation); + entityNotificationService.notifyCreateOrUpdateOrDeleteRelation(ctx.getTenantId(), null, + relation, ctx.getUser(), ActionType.RELATION_ADD_OR_UPDATE, null, relation); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index f2e61b8f2d..3225668004 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -19,14 +19,11 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @@ -52,14 +49,6 @@ public class AssetImportService extends BaseEntityImportService Date: Fri, 24 Jun 2022 13:57:06 +0300 Subject: [PATCH 73/80] Fix event for deleted relations on import --- .../sync/ie/importing/impl/BaseEntityImportService.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7caea242f8..d491571272 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -212,10 +212,8 @@ public abstract class BaseEntityImportService { - entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, - ActionType.RELATION_DELETED, null, existingRelation); - entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null, - ActionType.RELATION_DELETED, null, existingRelation); + entityNotificationService.notifyCreateOrUpdateOrDeleteRelation(tenantId, null, + existingRelation, ctx.getUser(), ActionType.RELATION_DELETED, null, existingRelation); }); } else if (Objects.equal(relation.getAdditionalInfo(), existingRelation.getAdditionalInfo())) { relationsMap.remove(relation); From 28e8865e742af984fa98bf66698ca55386cc5307 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jun 2022 14:51:36 +0300 Subject: [PATCH 74/80] Additional check that VC service is started --- .../java/org/thingsboard/server/msa/ContainerTestSuite.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 1c4b01c5d1..1fbf3755e6 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -47,6 +47,7 @@ public class ContainerTestSuite { private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; + private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; private static DockerComposeContainer testContainer; @@ -108,7 +109,9 @@ public class ContainerTestSuite { .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); + .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) + .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); } catch (Exception e) { log.error("Failed to create test container", e); fail("Failed to create test container"); From 8dc799709cf536ad9a3a2054bb13ec72a17ca581 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 24 Jun 2022 17:01:27 +0300 Subject: [PATCH 75/80] Fix entity relations import --- .../exporting/impl/DefaultEntityExportService.java | 8 ++++---- .../ie/importing/impl/BaseEntityImportService.java | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 7e236370df..a00a7c09aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; @@ -55,7 +55,7 @@ public class DefaultEntityExportService exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List relations = new ArrayList<>(); - List inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List inboundRelations = relationDao.findAllByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(inboundRelations); - List outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List outboundRelations = relationDao.findAllByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(outboundRelations); return relations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index d491571272..50349d0d1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -26,10 +26,8 @@ import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -49,13 +47,13 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; -import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -79,6 +77,8 @@ public abstract class BaseEntityImportService existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationDao.findAllByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationDao.findAllByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + // dao is used here instead of service to avoid getting cached values, because relationService.deleteRelation will evict value from cache only after transaction is committed for (EntityRelation existingRelation : existingRelations) { EntityRelation relation = relationsMap.get(existingRelation); if (relation == null) { importResult.setUpdatedRelatedEntities(true); - relationService.deleteRelation(tenantId, existingRelation); + relationService.deleteRelation(ctx.getTenantId(), existingRelation.getFrom(), existingRelation.getTo(), existingRelation.getType(), existingRelation.getTypeGroup()); importResult.addSendEventsCallback(() -> { entityNotificationService.notifyCreateOrUpdateOrDeleteRelation(tenantId, null, existingRelation, ctx.getUser(), ActionType.RELATION_DELETED, null, existingRelation); From 850aed8b82f55443c1e1998d01dff56997736536 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 24 Jun 2022 17:12:57 +0300 Subject: [PATCH 76/80] Add Swagger api params description for EntitiesVersionControlController --- .../controller/ControllerConstants.java | 1 + .../EntitiesVersionControlController.java | 27 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index 6a616671eb..a425b0968a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -138,6 +138,7 @@ public class ControllerConstants { protected static final String ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the entity version name."; protected static final String VERSION_ID_PARAM_DESCRIPTION = "Version id, for example fd82625bdd7d6131cf8027b44ee967012ecaf990. Represents commit hash."; + protected static final String BRANCH_PARAM_DESCRIPTION = "The name of the working branch, for example 'master'"; protected static final String MARKDOWN_CODE_BLOCK_START = "```json\n"; protected static final String MARKDOWN_CODE_BLOCK_END = "\n```"; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index dfefce4b12..2e37bcbe59 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -58,6 +58,9 @@ import java.util.List; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.controller.ControllerConstants.BRANCH_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; @@ -230,8 +233,11 @@ public class EntitiesVersionControlController extends BaseController { MARKDOWN_CODE_BLOCK_END + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"}) - public DeferredResult> listEntityVersions(@PathVariable EntityType entityType, + public DeferredResult> listEntityVersions(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, + @ApiParam(value = "A string value representing external entity id. This is `externalId` property of an entity, or otherwise if not set - simply id of this entity.") @PathVariable UUID externalEntityUuid, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION) @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @@ -256,7 +262,9 @@ public class EntitiesVersionControlController extends BaseController { "The response structure is the same as for `listEntityVersions` API method." + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"}) - public DeferredResult> listEntityTypeVersions(@PathVariable EntityType entityType, + public DeferredResult> listEntityTypeVersions(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @@ -279,7 +287,8 @@ public class EntitiesVersionControlController extends BaseController { "The response format is the same as for `listEntityVersions` API method." + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version", params = {"branch", "pageSize", "page"}) - public DeferredResult> listVersions(@RequestParam String branch, + public DeferredResult> listVersions(@ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) + @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -302,9 +311,11 @@ public class EntitiesVersionControlController extends BaseController { "Entities order will be the same as in the repository." + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"}) - public DeferredResult> listEntitiesAtVersion(@PathVariable EntityType entityType, + public DeferredResult> listEntitiesAtVersion(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); @@ -318,6 +329,7 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping(value = "/entity/{versionId}", params = {"branch"}) public DeferredResult> listAllEntitiesAtVersion(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); @@ -332,7 +344,9 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping("/info/{versionId}/{entityType}/{externalEntityUuid}") public DeferredResult getEntityDataInfo(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId, + @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable EntityType entityType, + @ApiParam(value = "A string value representing external entity id", required = true) @PathVariable UUID externalEntityUuid) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); @@ -344,8 +358,11 @@ public class EntitiesVersionControlController extends BaseController { "Entity data structure is the same as stored in a repository. " + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"}) - public DeferredResult compareEntityDataToVersion(@PathVariable EntityType entityType, + public DeferredResult compareEntityDataToVersion(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, + @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable UUID internalEntityUuid, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION) @RequestParam String branch, @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @RequestParam String versionId) throws Exception { From a83f99cfd9edbdbb94fd12fd8fba7bc29712f75c Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 24 Jun 2022 17:27:16 +0300 Subject: [PATCH 77/80] Revert redudant logging in TbAbstractGetAttributesNode. --- .../rule/engine/metadata/TbAbstractGetAttributesNode.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index f5752154c9..47d6af8e9c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonParseException; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.thingsboard.rule.engine.api.TbContext; @@ -50,7 +49,6 @@ import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; -@Slf4j public abstract class TbAbstractGetAttributesNode implements TbNode { private static ObjectMapper mapper = new ObjectMapper(); @@ -113,7 +111,6 @@ public abstract class TbAbstractGetAttributesNode> attributeKvEntryListFuture = ctx.getAttributesService().find(ctx.getTenantId(), entityId, scope, keys); return Futures.transform(attributeKvEntryListFuture, attributeKvEntryList -> { - log.warn("[{}][{}][{}][{}] Lookup attribute result: {}", ctx.getTenantId(), entityId, scope, keys, attributeKvEntryList); if (!CollectionUtils.isEmpty(attributeKvEntryList)) { List existingAttributesKvEntry = attributeKvEntryList.stream().filter(attributeKvEntry -> keys.contains(attributeKvEntry.getKey())).collect(Collectors.toList()); existingAttributesKvEntry.forEach(kvEntry -> msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString())); From e9c9e24e2f1f036db73daacb933bf0f788ec7558 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jun 2022 10:35:50 +0300 Subject: [PATCH 78/80] Improvements to the chunks processing --- .../DefaultGitVersionControlQueueService.java | 73 +++++++++---------- .../vc/GitVersionControlQueueService.java | 2 - common/cluster-api/src/main/proto/queue.proto | 11 --- .../DefaultClusterVersionControlService.java | 8 -- 4 files changed, 35 insertions(+), 59 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 5a726c871f..6eaa2df74f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -79,6 +79,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -125,7 +126,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build() - ).build(), wrap(future, commit, commit.getRequestId())); + ).build(), wrap(future, commit)); return future; } @@ -151,7 +152,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setChunkedMsgId(chunkedMsgId).setChunkIndex(chunkIndex.getAndIncrement()) .setChunksCount(chunksCount).build() ).build() - ).build(), wrap(chunkFuture, null, commit.getRequestId())); + ).build(), wrap(chunkFuture, null)); futures.add(chunkFuture); }); return Futures.transform(Futures.allAsList(futures), r -> { @@ -170,7 +171,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu buildCommitRequest(commit).setDeleteMsg( TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() ).build() - ).build(), wrap(future, null, commit.getRequestId())); + ).build(), wrap(future, null)); return future; } @@ -280,18 +281,11 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .build())); } - @Override - public ListenableFuture getContentsDiff(TenantId tenantId, String content1, String content2) { - ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2); - return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder() - .setContent1(content1) - .setContent2(content2))); - } - @Override @SuppressWarnings("rawtypes") public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); + chunkedMsgs.put(request.getRequestId(), new HashMap<>()); registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() .setVersionId(versionId) .setEntityType(entityId.getEntityType().name()) @@ -314,9 +308,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); if (request.getTimeoutTask() == null) { - request.setTimeoutTask(scheduler.schedule(() -> { - processTimeout(request.getRequestId()); - }, requestTimeout, TimeUnit.MILLISECONDS)); + request.setTimeoutTask(scheduler.schedule(() -> processTimeout(request.getRequestId()), requestTimeout, TimeUnit.MILLISECONDS)); } } else { throw new RuntimeException("Future is already done!"); @@ -335,7 +327,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); - + chunkedMsgs.put(request.getRequestId(), new HashMap<>()); registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() .setVersionId(versionId) .setEntityType(entityType.name()) @@ -418,14 +410,11 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu ((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse)); } else if (vcResponseMsg.hasEntityContentResponse()) { TransportProtos.EntityContentResponseMsg responseMsg = vcResponseMsg.getEntityContentResponse(); - String[] msgChunks = chunkedMsgs.computeIfAbsent(requestId, id -> new HashMap<>()) - .computeIfAbsent(responseMsg.getChunkedMsgId(), id -> new String[responseMsg.getChunksCount()]); - msgChunks[responseMsg.getChunkIndex()] = responseMsg.getData(); log.trace("[{}] received chunk {} for 'getEntity'", responseMsg.getChunkedMsgId(), responseMsg.getChunkIndex()); - if (CollectionsUtil.countNonNull(msgChunks) == responseMsg.getChunksCount()) { + var joined = joinChunks(requestId, responseMsg, 1); + if (joined.isPresent()) { log.trace("[{}] collected all chunks for 'getEntity'", responseMsg.getChunkedMsgId()); - String data = String.join("", msgChunks); - ((EntityContentGitRequest) request).getFuture().set(toData(data)); + ((EntityContentGitRequest) request).getFuture().set(joined.get().get(0)); } else { completed = false; } @@ -433,17 +422,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu TransportProtos.EntitiesContentResponseMsg responseMsg = vcResponseMsg.getEntitiesContentResponse(); TransportProtos.EntityContentResponseMsg item = responseMsg.getItem(); if (responseMsg.getItemsCount() > 0) { - Map chunkedItems = chunkedMsgs.computeIfAbsent(requestId, id -> new HashMap<>()); - String[] itemChunks = chunkedItems.computeIfAbsent(item.getChunkedMsgId(), id -> { - return new String[item.getChunksCount()]; - }); - itemChunks[item.getChunkIndex()] = item.getData(); - if (chunkedItems.size() == responseMsg.getItemsCount() && chunkedItems.values().stream() - .allMatch(chunks -> CollectionsUtil.countNonNull(chunks) == chunks.length)) { - ((EntitiesContentGitRequest) request).getFuture().set(chunkedItems.values().stream() - .map(chunks -> String.join("", chunks)) - .map(this::toData) - .collect(Collectors.toList())); + var joined = joinChunks(requestId, item, responseMsg.getItemsCount()); + if (joined.isPresent()) { + ((EntitiesContentGitRequest) request).getFuture().set(joined.get()); } else { completed = false; } @@ -464,9 +445,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .build()) .collect(Collectors.toList()); ((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList); - } else if (vcResponseMsg.hasContentsDiffResponse()) { - String diff = vcResponseMsg.getContentsDiffResponse().getDiff(); - ((ContentsDiffGitRequest) request).getFuture().set(diff); } } if (completed) { @@ -474,6 +452,25 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } } + @SuppressWarnings("rawtypes") + private Optional> joinChunks(UUID requestId, TransportProtos.EntityContentResponseMsg responseMsg, int expectedMsgCount) { + var chunksMap = chunkedMsgs.get(requestId); + if (chunksMap == null) { + return Optional.empty(); + } + String[] msgChunks = chunksMap.computeIfAbsent(responseMsg.getChunkedMsgId(), id -> new String[responseMsg.getChunksCount()]); + msgChunks[responseMsg.getChunkIndex()] = responseMsg.getData(); + if (chunksMap.size() == expectedMsgCount && chunksMap.values().stream() + .allMatch(chunks -> CollectionsUtil.countNonNull(chunks) == chunks.length)) { + return Optional.of(chunksMap.values().stream() + .map(chunks -> String.join("", chunks)) + .map(this::toData) + .collect(Collectors.toList())); + } else { + return Optional.empty(); + } + } + private void processTimeout(UUID requestId) { PendingGitRequest pendingRequest = removePendingRequest(requestId); if (pendingRequest != null) { @@ -515,6 +512,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return JacksonUtil.fromString(data, EntityExportData.class); } + //The future will be completed when the corresponding result arrives from kafka private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override @@ -528,18 +526,17 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu }; } - private TbQueueCallback wrap(SettableFuture future, T value, UUID requestId) { + //The future will be completed when the request is successfully sent to kafka + private TbQueueCallback wrap(SettableFuture future, T value) { return new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { future.set(value); - removePendingRequest(requestId); } @Override public void onFailure(Throwable t) { future.setException(t); - removePendingRequest(requestId); } }; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index a1e64da8f1..cc83479896 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -64,8 +64,6 @@ public interface GitVersionControlQueueService { ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2); - ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); - ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings); ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 855ddb9299..1b6bc43f74 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -829,15 +829,6 @@ message EntityVersionsDiff { string rawDiff = 6; } -message ContentsDiffRequestMsg { - string content1 = 1; - string content2 = 2; -} - -message ContentsDiffResponseMsg { - string diff = 1; -} - message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -859,7 +850,6 @@ message ToVersionControlServiceMsg { EntityContentRequestMsg entityContentRequest = 14; EntitiesContentRequestMsg entitiesContentRequest = 15; VersionsDiffRequestMsg versionsDiffRequest = 16; - ContentsDiffRequestMsg contentsDiffRequest = 17; } message VersionControlResponseMsg { @@ -874,7 +864,6 @@ message VersionControlResponseMsg { EntityContentResponseMsg entityContentResponse = 9; EntitiesContentResponseMsg entitiesContentResponse = 10; VersionsDiffResponseMsg versionsDiffResponse = 11; - ContentsDiffResponseMsg contentsDiffResponse = 12; } /** diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index d8d137ae5b..7d8e81d1c4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -257,8 +257,6 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); } else if (msg.hasVersionsDiffRequest()) { handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest()); - } else if (msg.hasContentsDiffRequest()) { - handleContentsDiffRequest(ctx, msg.getContentsDiffRequest()); } } } @@ -394,12 +392,6 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .addAllDiff(diffList))); } - private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException { - String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2()); - reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder() - .setDiff(diff))); - } - private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); From 48c3ce70476536f3a4ba17de25a9bb70343fde9f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jun 2022 11:48:58 +0300 Subject: [PATCH 79/80] Fix cache value serialization issue --- .../sync/vc/DefaultEntitiesVersionControlService.java | 2 ++ .../server/cache/SimpleTbCacheValueWrapper.java | 2 ++ .../server/common/data/sync/vc/EntityLoadError.java | 5 ++++- .../server/common/data/sync/vc/EntityTypeLoadResult.java | 6 +++++- docker/.gitignore | 8 ++++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 74c5c78dae..5bfc696cc1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -174,9 +174,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private T getStatus(SecurityUser user, UUID requestId, Function getter) throws ThingsboardException { var cacheEntry = taskCache.get(requestId); if (cacheEntry == null || cacheEntry.get() == null) { + log.debug("[{}] No cache record: {}", requestId, cacheEntry); throw new ThingsboardException(ThingsboardErrorCode.ITEM_NOT_FOUND); } else { var entry = cacheEntry.get(); + log.debug("[{}] Cache get: {}", requestId, entry); var result = getter.apply(entry); if (result == null) { throw new ThingsboardException(ThingsboardErrorCode.BAD_REQUEST_PARAMS); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java b/common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java index f5607fb760..58ccfd8fb9 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java @@ -17,8 +17,10 @@ package org.thingsboard.server.cache; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.ToString; import org.springframework.cache.Cache; +@ToString @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class SimpleTbCacheValueWrapper implements TbCacheValueWrapper { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java index bc3475c89e..cd3dc5e0e2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -20,12 +20,15 @@ import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; +import java.io.Serializable; import java.util.List; @Data @Builder @JsonInclude(JsonInclude.Include.NON_NULL) -public class EntityLoadError { +public class EntityLoadError implements Serializable { + + private static final long serialVersionUID = 7538450180582109391L; private String type; private EntityId source; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java index 84a28d0770..06df7ebb83 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -21,11 +21,15 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; +import java.io.Serializable; + @Data @AllArgsConstructor @NoArgsConstructor @Builder -public class EntityTypeLoadResult { +public class EntityTypeLoadResult implements Serializable { + private static final long serialVersionUID = -8428039809651395241L; + private EntityType entityType; private int created; private int updated; diff --git a/docker/.gitignore b/docker/.gitignore index 9c4c778f28..c9172ae6ce 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -6,4 +6,12 @@ tb-node/postgres/** tb-node/cassandra/** tb-transports/*/log tb-vc-executor/log/** +tb-node/redis-cluster-data-0/** +tb-node/redis-cluster-data-1/** +tb-node/redis-cluster-data-2/** +tb-node/redis-cluster-data-3/** +tb-node/redis-cluster-data-4/** +tb-node/redis-cluster-data-5/** +tb-node/redis-data/** + !.env From 24e856516ff6d9b96112287968b5e350ed38a4b7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jun 2022 12:33:02 +0300 Subject: [PATCH 80/80] Fix integration test --- .../sync/ie/importing/impl/DeviceProfileImportService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java index 2c27f6941f..b1189412f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -69,7 +70,8 @@ public class DeviceProfileImportService extends BaseEntityImportService