tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Implemented MIN/MAX/AVG/SUM/COUNT calculation for mixed number data types

pull/1445/head
Andrew Shvayka 8 years ago
parent
66c0e7179e
commit
7358b4a79b
5 changed files with 142 additions and 27 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 35
      dao/src/main/java/org/thingsboard/server/dao/model/sql/TsKvEntity.java
  2. 6
      dao/src/main/java/org/thingsboard/server/dao/sql/timeseries/TsKvRepository.java
  3. 20
      dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java
  4. 4
      dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java
  5. 104
      dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java

35
dao/src/main/java/org/thingsboard/server/dao/model/sql/TsKvEntity.java
View File

@ -52,22 +52,33 @@ public final class TsKvEntity implements ToData<TsKvEntry> {
public TsKvEntity() {
}
public TsKvEntity(Double avgLongValue, Double avgDoubleValue) {
if(avgLongValue != null) {
this.longValue = avgLongValue.longValue();
public TsKvEntity(Long longSumValue, Double doubleSumValue, Long longCountValue, Long doubleCountValue) {
double sum = 0.0;
if (longSumValue != null) {
sum += longSumValue;
}
this.doubleValue = avgDoubleValue;
if (doubleSumValue != null) {
sum += doubleSumValue;
}
this.doubleValue = sum / (longCountValue + doubleCountValue);
}
public TsKvEntity(Long sumLongValue, Double sumDoubleValue) {
this.longValue = sumLongValue;
this.doubleValue = sumDoubleValue;
if (sumDoubleValue != null) {
this.doubleValue = sumDoubleValue + (sumLongValue != null ? sumLongValue.doubleValue() : 0.0);
} else {
this.longValue = sumLongValue;
}
}
public TsKvEntity(String strValue, Long longValue, Double doubleValue) {
public TsKvEntity(String strValue, Long longValue, Double doubleValue, boolean max) {
this.strValue = strValue;
this.longValue = longValue;
this.doubleValue = doubleValue;
if (longValue != null && doubleValue != null) {
this.doubleValue = max ? Math.max(doubleValue, longValue.doubleValue()) : Math.min(doubleValue, longValue.doubleValue());
} else {
this.longValue = longValue;
this.doubleValue = doubleValue;
}
}
public TsKvEntity(Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount) {
@ -75,10 +86,8 @@ public final class TsKvEntity implements ToData<TsKvEntry> {
this.longValue = booleanValueCount;
} else if (strValueCount != 0) {
this.longValue = strValueCount;
} else if (longValueCount != 0) {
this.longValue = longValueCount;
} else if (doubleValueCount != 0) {
this.longValue = doubleValueCount;
} else {
this.longValue = longValueCount + doubleValueCount;
}
}

6
dao/src/main/java/org/thingsboard/server/dao/sql/timeseries/TsKvRepository.java
View File

@ -55,7 +55,7 @@ public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvComposite
@Param("endTs") long endTs);
@Async
@Query("SELECT new TsKvEntity(MAX(tskv.strValue), MAX(tskv.longValue), MAX(tskv.doubleValue)) FROM TsKvEntity tskv " +
@Query("SELECT new TsKvEntity(MAX(tskv.strValue), MAX(tskv.longValue), MAX(tskv.doubleValue), true) FROM TsKvEntity tskv " +
"WHERE tskv.entityId = :entityId AND tskv.entityType = :entityType " +
"AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts < :endTs")
CompletableFuture<TsKvEntity> findMax(@Param("entityId") String entityId,
@ -65,7 +65,7 @@ public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvComposite
@Param("endTs") long endTs);
@Async
@Query("SELECT new TsKvEntity(MIN(tskv.strValue), MIN(tskv.longValue), MIN(tskv.doubleValue)) FROM TsKvEntity tskv " +
@Query("SELECT new TsKvEntity(MIN(tskv.strValue), MIN(tskv.longValue), MIN(tskv.doubleValue), false) FROM TsKvEntity tskv " +
"WHERE tskv.entityId = :entityId AND tskv.entityType = :entityType " +
"AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts < :endTs")
CompletableFuture<TsKvEntity> findMin(@Param("entityId") String entityId,
@ -85,7 +85,7 @@ public interface TsKvRepository extends CrudRepository<TsKvEntity, TsKvComposite
@Param("endTs") long endTs);
@Async
@Query("SELECT new TsKvEntity(AVG(tskv.longValue), AVG(tskv.doubleValue)) FROM TsKvEntity tskv " +
@Query("SELECT new TsKvEntity(SUM(tskv.longValue), SUM(tskv.doubleValue), COUNT(tskv.longValue), COUNT(tskv.doubleValue)) FROM TsKvEntity tskv " +
"WHERE tskv.entityId = :entityId AND tskv.entityType = :entityType " +
"AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts < :endTs")
CompletableFuture<TsKvEntity> findAvg(@Param("entityId") String entityId,

20
dao/src/main/java/org/thingsboard/server/dao/timeseries/AggregatePartitionsFunction.java
View File

@ -98,6 +98,7 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct
curLValue = getLongValue(row);
}
if (doubleCount > 0) {
aggResult.hasDouble = true;
aggResult.dataType = DataType.DOUBLE;
curCount += doubleCount;
curDValue = getDoubleValue(row);
@ -222,17 +223,25 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct
if (aggResult.count == 0 || (aggResult.dataType == DataType.DOUBLE && aggResult.dValue == null) || (aggResult.dataType == DataType.LONG && aggResult.lValue == null)) {
return Optional.empty();
} else if (aggResult.dataType == DataType.DOUBLE || aggResult.dataType == DataType.LONG) {
double sum = Optional.ofNullable(aggResult.dValue).orElse(0.0d) + Optional.ofNullable(aggResult.lValue).orElse(0L);
return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? sum : (sum / aggResult.count))));
if(aggregation == Aggregation.AVG || aggResult.hasDouble) {
double sum = Optional.ofNullable(aggResult.dValue).orElse(0.0d) + Optional.ofNullable(aggResult.lValue).orElse(0L);
return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.SUM ? sum : (sum / aggResult.count))));
} else {
return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggregation == Aggregation.SUM ? aggResult.lValue : (aggResult.lValue / aggResult.count))));
}
}
return Optional.empty();
}
private Optional<TsKvEntry> processMinOrMaxResult(AggregationResult aggResult) {
if (aggResult.dataType == DataType.DOUBLE || aggResult.dataType == DataType.LONG) {
double currentD = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.dValue).orElse(Double.MAX_VALUE) : Optional.ofNullable(aggResult.dValue).orElse(Double.MIN_VALUE);
double currentL = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.lValue).orElse(Long.MAX_VALUE) : Optional.ofNullable(aggResult.lValue).orElse(Long.MIN_VALUE);
return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.MIN ? Math.min(currentD, currentL) : Math.max(currentD, currentL))));
if(aggResult.hasDouble) {
double currentD = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.dValue).orElse(Double.MAX_VALUE) : Optional.ofNullable(aggResult.dValue).orElse(Double.MIN_VALUE);
double currentL = aggregation == Aggregation.MIN ? Optional.ofNullable(aggResult.lValue).orElse(Long.MAX_VALUE) : Optional.ofNullable(aggResult.lValue).orElse(Long.MIN_VALUE);
return Optional.of(new BasicTsKvEntry(ts, new DoubleDataEntry(key, aggregation == Aggregation.MIN ? Math.min(currentD, currentL) : Math.max(currentD, currentL))));
} else {
return Optional.of(new BasicTsKvEntry(ts, new LongDataEntry(key, aggResult.lValue)));
}
} else if (aggResult.dataType == DataType.STRING) {
return Optional.of(new BasicTsKvEntry(ts, new StringDataEntry(key, aggResult.sValue)));
} else {
@ -247,5 +256,6 @@ public class AggregatePartitionsFunction implements com.google.common.base.Funct
Double dValue = null;
Long lValue = null;
long count = 0;
boolean hasDouble = false;
}
}

4
dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java
View File

@ -25,9 +25,7 @@ import java.util.Arrays;
@RunWith(ClasspathSuite.class)
@ClassnameFilters({
"org.thingsboard.server.dao.service.*ServiceNoSqlTest",
"org.thingsboard.server.dao.service.queue.cassandra.*.*.*Test",
"org.thingsboard.server.dao.service.queue.cassandra.*Test"
"org.thingsboard.server.dao.service.*ServiceNoSqlTest"
})
public class NoSqlDaoServiceTestSuite {

104
dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java
View File

@ -221,13 +221,13 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
60000, 20000, 3, Aggregation.AVG))).get();
assertEquals(3, list.size());
assertEquals(10000, list.get(0).getTs());
assertEquals(java.util.Optional.of(150L), list.get(0).getLongValue());
assertEquals(java.util.Optional.of(150.0), list.get(0).getDoubleValue());
assertEquals(30000, list.get(1).getTs());
assertEquals(java.util.Optional.of(350L), list.get(1).getLongValue());
assertEquals(java.util.Optional.of(350.0), list.get(1).getDoubleValue());
assertEquals(50000, list.get(2).getTs());
assertEquals(java.util.Optional.of(550L), list.get(2).getLongValue());
assertEquals(java.util.Optional.of(550.0), list.get(2).getDoubleValue());
list = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery(LONG_KEY, 0,
60000, 20000, 3, Aggregation.SUM))).get();
@ -282,12 +282,110 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
assertEquals(java.util.Optional.of(2L), list.get(2).getLongValue());
}
@Test
public void testFindDeviceLongAndDoubleTsData() throws Exception {
DeviceId deviceId = new DeviceId(UUIDs.timeBased());
List<TsKvEntry> entries = new ArrayList<>();
entries.add(save(deviceId, 5000, 100));
entries.add(save(deviceId, 15000, 200.0));
entries.add(save(deviceId, 25000, 300));
entries.add(save(deviceId, 35000, 400.0));
entries.add(save(deviceId, 45000, 500));
entries.add(save(deviceId, 55000, 600.0));
List<TsKvEntry> list = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery(LONG_KEY, 0,
60000, 20000, 3, Aggregation.NONE))).get();
assertEquals(3, list.size());
assertEquals(55000, list.get(0).getTs());
assertEquals(java.util.Optional.of(600.0), list.get(0).getDoubleValue());
assertEquals(45000, list.get(1).getTs());
assertEquals(java.util.Optional.of(500L), list.get(1).getLongValue());
assertEquals(35000, list.get(2).getTs());
assertEquals(java.util.Optional.of(400.0), list.get(2).getDoubleValue());
list = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery(LONG_KEY, 0,
60000, 20000, 3, Aggregation.AVG))).get();
assertEquals(3, list.size());
assertEquals(10000, list.get(0).getTs());
assertEquals(java.util.Optional.of(150.0), list.get(0).getDoubleValue());
assertEquals(30000, list.get(1).getTs());
assertEquals(java.util.Optional.of(350.0), list.get(1).getDoubleValue());
assertEquals(50000, list.get(2).getTs());
assertEquals(java.util.Optional.of(550.0), list.get(2).getDoubleValue());
list = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery(LONG_KEY, 0,
60000, 20000, 3, Aggregation.SUM))).get();
assertEquals(3, list.size());
assertEquals(10000, list.get(0).getTs());
assertEquals(java.util.Optional.of(300.0), list.get(0).getDoubleValue());
assertEquals(30000, list.get(1).getTs());
assertEquals(java.util.Optional.of(700.0), list.get(1).getDoubleValue());
assertEquals(50000, list.get(2).getTs());
assertEquals(java.util.Optional.of(1100.0), list.get(2).getDoubleValue());
list = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery(LONG_KEY, 0,
60000, 20000, 3, Aggregation.MIN))).get();
assertEquals(3, list.size());
assertEquals(10000, list.get(0).getTs());
assertEquals(java.util.Optional.of(100.0), list.get(0).getDoubleValue());
assertEquals(30000, list.get(1).getTs());
assertEquals(java.util.Optional.of(300.0), list.get(1).getDoubleValue());
assertEquals(50000, list.get(2).getTs());
assertEquals(java.util.Optional.of(500.0), list.get(2).getDoubleValue());
list = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery(LONG_KEY, 0,
60000, 20000, 3, Aggregation.MAX))).get();
assertEquals(3, list.size());
assertEquals(10000, list.get(0).getTs());
assertEquals(java.util.Optional.of(200.0), list.get(0).getDoubleValue());
assertEquals(30000, list.get(1).getTs());
assertEquals(java.util.Optional.of(400.0), list.get(1).getDoubleValue());
assertEquals(50000, list.get(2).getTs());
assertEquals(java.util.Optional.of(600.0), list.get(2).getDoubleValue());
list = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery(LONG_KEY, 0,
60000, 20000, 3, Aggregation.COUNT))).get();
assertEquals(3, list.size());
assertEquals(10000, list.get(0).getTs());
assertEquals(java.util.Optional.of(2L), list.get(0).getLongValue());
assertEquals(30000, list.get(1).getTs());
assertEquals(java.util.Optional.of(2L), list.get(1).getLongValue());
assertEquals(50000, list.get(2).getTs());
assertEquals(java.util.Optional.of(2L), list.get(2).getLongValue());
}
private TsKvEntry save(DeviceId deviceId, long ts, long value) throws Exception {
TsKvEntry entry = new BasicTsKvEntry(ts, new LongDataEntry(LONG_KEY, value));
tsService.save(tenantId, deviceId, entry).get();
return entry;
}
private TsKvEntry save(DeviceId deviceId, long ts, double value) throws Exception {
TsKvEntry entry = new BasicTsKvEntry(ts, new DoubleDataEntry(LONG_KEY, value));
tsService.save(tenantId, deviceId, entry).get();
return entry;
}
private void saveEntries(DeviceId deviceId, long ts) throws ExecutionException, InterruptedException {
tsService.save(tenantId, deviceId, toTsEntry(ts, stringKvEntry)).get();
tsService.save(tenantId, deviceId, toTsEntry(ts, longKvEntry)).get();

Write Preview
Loading…
Cancel
Save