From 123ea76c94718c4eaf251a738887727ad2564abb Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 3 Nov 2020 18:59:12 +0200 Subject: [PATCH] Improvements for data points storage --- .../DefaultTbApiUsageStateService.java | 30 ++++---- .../DefaultRuleEngineStatisticsService.java | 8 +-- .../AbstractSubscriptionService.java | 6 +- .../DefaultTelemetrySubscriptionService.java | 68 +++++++++++++------ .../telemetry/InternalTelemetryService.java | 2 + .../dao/timeseries/TimeseriesService.java | 4 +- .../server/common/data/kv/BasicTsKvEntry.java | 19 +++++- .../server/common/data/kv/TsKvEntry.java | 5 ++ ...stractChunkedAggregationTimeseriesDao.java | 2 +- .../dao/sqlts/AbstractSqlTimeseriesDao.java | 21 ++++++ .../dao/sqlts/hsql/JpaHsqlTimeseriesDao.java | 7 +- .../dao/sqlts/psql/JpaPsqlTimeseriesDao.java | 7 +- .../timescale/TimescaleTimeseriesDao.java | 11 +-- .../dao/tenant/TenantProfileServiceImpl.java | 2 +- .../dao/timeseries/BaseTimeseriesService.java | 34 +++++++--- .../CassandraBaseTimeseriesDao.java | 19 ++++-- .../server/dao/timeseries/TimeseriesDao.java | 4 +- .../usagerecord/ApiUsageStateServiceImpl.java | 2 +- .../dao/service/BaseEntityServiceTest.java | 4 +- 19 files changed, 181 insertions(+), 74 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index 88868d2258..42f980909a 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.apiusage; import com.google.common.util.concurrent.FutureCallback; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ApiUsageRecordKey; @@ -51,7 +53,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.profile.TbTenantProfileCache; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.InternalTelemetryService; -import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.PostConstruct; import java.util.ArrayList; @@ -73,20 +74,25 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public static final String HOURLY = "Hourly"; public static final FutureCallback VOID_CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Integer result) {} + public void onSuccess(@Nullable Integer result) { + } @Override - public void onFailure(Throwable t) {} + public void onFailure(Throwable t) { + } }; private final TbClusterService clusterService; private final PartitionService partitionService; private final TenantService tenantService; - private final ApiUsageStateService apiUsageStateService; private final TimeseriesService tsService; - private final InternalTelemetryService tsWsService; + private final ApiUsageStateService apiUsageStateService; private final SchedulerComponent scheduler; private final TbTenantProfileCache tenantProfileCache; + @Lazy + @Autowired + private InternalTelemetryService tsWsService; + // Tenants that should be processed on this server private final Map myTenantStates = new ConcurrentHashMap<>(); // Tenants that should be processed on other servers @@ -102,16 +108,16 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { public DefaultTbApiUsageStateService(TbClusterService clusterService, PartitionService partitionService, - TenantService tenantService, ApiUsageStateService apiUsageStateService, - TimeseriesService tsService, TelemetrySubscriptionService tsWsService, + TenantService tenantService, + TimeseriesService tsService, + ApiUsageStateService apiUsageStateService, SchedulerComponent scheduler, TbTenantProfileCache tenantProfileCache) { this.clusterService = clusterService; this.partitionService = partitionService; this.tenantService = tenantService; - this.apiUsageStateService = apiUsageStateService; this.tsService = tsService; - this.tsWsService = tsWsService; + this.apiUsageStateService = apiUsageStateService; this.scheduler = scheduler; this.tenantProfileCache = tenantProfileCache; } @@ -156,7 +162,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } finally { updateLock.unlock(); } - tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK); if (!result.isEmpty()) { persistAndNotify(tenantState, result); } @@ -256,7 +262,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { } } if (!profileThresholds.isEmpty()) { - tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK); } } @@ -266,7 +272,7 @@ public class DefaultTbApiUsageStateService implements TbApiUsageStateService { long ts = System.currentTimeMillis(); List stateTelemetry = new ArrayList<>(); result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new BooleanDataEntry(apiFeature.getApiStateKey(), aState))))); - tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, 0L, VOID_CALLBACK); + tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK); } private void checkStartOfNextCycle() { diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index b46033389b..a832eb0870 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -48,9 +48,9 @@ import java.util.stream.Collectors; public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; - public static final FutureCallback CALLBACK = new FutureCallback() { + public static final FutureCallback CALLBACK = new FutureCallback() { @Override - public void onSuccess(@Nullable Void result) { + public void onSuccess(@Nullable Integer result) { } @@ -85,7 +85,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) .collect(Collectors.toList()); if (!tsList.isEmpty()) { - tsService.saveAndNotify(tenantId, serviceAssetId, tsList, CALLBACK); + tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK); } } } catch (DataValidationException e) { @@ -97,7 +97,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { TsKvEntry tsKv = new BasicTsKvEntry(ts, new JsonDataEntry("ruleEngineException", e.toJsonString())); try { - tsService.saveAndNotify(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); } catch (DataValidationException e2) { if (!e2.getMessage().equalsIgnoreCase("Asset is referencing to non-existent tenant!")) { throw e2; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 4178502c77..468c3c26b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -105,10 +105,10 @@ public abstract class AbstractSubscriptionService implements ApplicationListener } } - protected void addWsCallback(ListenableFuture> saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback>() { + protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback() { @Override - public void onSuccess(@Nullable List result) { + public void onSuccess(@Nullable T result) { callback.accept(null); } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 53d0e08da2..7a35e6b317 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -40,9 +40,11 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -71,6 +73,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TimeseriesService tsService; private final EntityViewService entityViewService; private final TbApiUsageClient apiUsageClient; + private final TbApiUsageStateService apiUsageStateService; private ExecutorService tsCallBackExecutor; @@ -79,12 +82,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer EntityViewService entityViewService, TbClusterService clusterService, PartitionService partitionService, - TbApiUsageClient apiUsageClient) { + TbApiUsageClient apiUsageClient, + TbApiUsageStateService apiUsageStateService) { super(clusterService, partitionService); this.attrService = attrService; this.tsService = tsService; this.entityViewService = entityViewService; this.apiUsageClient = apiUsageClient; + this.apiUsageStateService = apiUsageStateService; } @PostConstruct @@ -114,25 +119,34 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { checkInternalEntity(entityId); - saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback() { - @Override - public void onSuccess(Integer result) { - if (result != null && result > 0) { - apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + if (apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + saveAndNotifyInternal(tenantId, entityId, ts, ttl, new FutureCallback() { + @Override + public void onSuccess(Integer result) { + if (result != null && result > 0) { + apiUsageClient.report(tenantId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + } + callback.onSuccess(null); } - callback.onSuccess(null); - } - @Override - public void onFailure(Throwable t) { - callback.onFailure(t); - } - }); + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); + } else{ + callback.onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); + } + } + + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { + saveAndNotifyInternal(tenantId, entityId, ts, 0L, callback); } @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { - ListenableFuture> saveFuture = tsService.save(tenantId, entityId, ts, ttl); + ListenableFuture saveFuture = tsService.save(tenantId, entityId, ts, ttl); addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { @@ -197,7 +211,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } @@ -210,7 +224,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); - addMainCallback(saveFuture, callback); + addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } @@ -223,7 +237,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); } @@ -236,7 +250,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { ListenableFuture> deleteFuture = tsService.removeLatest(tenantId, entityId, keys); - addMainCallback(deleteFuture, callback); + addVoidCallback(deleteFuture, callback); } @Override @@ -321,7 +335,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + private void addVoidCallback(ListenableFuture saveFuture, final FutureCallback callback) { Futures.addCallback(saveFuture, new FutureCallback() { @Override public void onSuccess(@Nullable S result) { @@ -335,6 +349,20 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }, tsCallBackExecutor); } + private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { + Futures.addCallback(saveFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable S result) { + callback.onSuccess(result); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }, tsCallBackExecutor); + } + private void checkInternalEntity(EntityId entityId) { if (EntityType.API_USAGE_STATE.equals(entityId.getEntityType())) { throw new RuntimeException("Can't update API Usage State!"); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 53cc23f23a..1ca2cdeacb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -29,6 +29,8 @@ import java.util.List; */ public interface InternalTelemetryService extends RuleEngineTelemetryService { + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 81dedff132..0e11aedaf5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -36,9 +36,9 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java index c2d6688004..bcf49899dc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -19,7 +19,7 @@ import java.util.Objects; import java.util.Optional; public class BasicTsKvEntry implements TsKvEntry { - + private static final int MAX_CHARS_PER_DATA_POINT = 512; private final long ts; private final KvEntry kv; @@ -99,4 +99,21 @@ public class BasicTsKvEntry implements TsKvEntry { public String getValueAsString() { return kv.getValueAsString(); } + + @Override + public int getDataPoints() { + int length; + switch (getDataType()) { + case STRING: + length = getStrValue().get().length(); + break; + case JSON: + length = getJsonValue().get().length(); + break; + default: + return 1; + } + return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index 315d3940df..e2f8eaf1e5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * Represents time series KV data entry * @@ -25,4 +27,7 @@ public interface TsKvEntry extends KvEntry { long getTs(); + @JsonIgnore + int getDataPoints(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 3cee730ee3..616260ba45 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -100,7 +100,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { return Futures.immediateFuture(null); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 8dcefeca61..c585804b82 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -30,11 +30,14 @@ import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { + protected static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); + @Autowired protected ScheduledLogExecutorComponent logExecutor; @@ -56,6 +59,9 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries @Value("${sql.batch_sort:false}") protected boolean batchSortEnabled; + @Value("${sql.ttl.ts.ts_key_value_ttl:0}") + private long systemTtl; + protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries .stream() @@ -75,4 +81,19 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries } }, service); } + + protected long computeTtl(long ttl) { + if (systemTtl > 0) { + if (ttl == 0) { + ttl = systemTtl; + } else { + ttl = Math.min(systemTtl, ttl); + } + } + return ttl; + } + + protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { + return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java index 5ea387328d..025134e187 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.dao.sqlts.hsql; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; @@ -35,7 +37,8 @@ import org.thingsboard.server.dao.util.SqlTsDao; public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao implements TimeseriesDao { @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TsKvEntity entity = new TsKvEntity(); @@ -48,7 +51,7 @@ public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java index 8c50bcb38c..3d971f61c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.dao.sqlts.psql; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -71,7 +73,8 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); savePartitionIfNotExist(tsKvEntry.getTs()); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); @@ -85,7 +88,7 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } private void savePartitionIfNotExist(long ts) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 6d5923c508..f6fb027d7e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -101,7 +102,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); Integer keyId = getOrSaveKeyId(strKey); TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); @@ -113,14 +115,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - log.trace("Saving entity to timescale db: {}", entity); - return tsQueue.add(entity); + return Futures.transform(tsQueue.add(entity), v -> dataPointDays, MoreExecutors.directExecutor()); } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - return Futures.immediateFuture(null); + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + return Futures.immediateFuture(0); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 5bafe4f799..95dd0ab45a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index b86d6128f6..3b97e2bb82 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.timeseries; +import com.google.common.base.Function; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -55,6 +57,20 @@ public class BaseTimeseriesService implements TimeseriesService { private static final int INSERTS_PER_ENTRY = 3; private static final int DELETES_PER_ENTRY = INSERTS_PER_ENTRY; + public static final Function, Integer> SUM_ALL_INTEGERS = new Function, Integer>() { + @Override + public @Nullable Integer apply(@Nullable List input) { + int result = 0; + if (input != null) { + for (Integer tmp : input) { + if (tmp != null) { + result += tmp; + } + } + } + return result; + } + }; @Value("${database.ts_max_intervals}") private long maxTsIntervals; @@ -101,26 +117,26 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); if (tsKvEntry == null) { throw new IncorrectParameterException("Key value entry can't be null"); } - List> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); + List> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.allAsList(futures); + return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * INSERTS_PER_ENTRY); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * INSERTS_PER_ENTRY); for (TsKvEntry tsKvEntry : tsKvEntries) { if (tsKvEntry == null) { throw new IncorrectParameterException("Key value entry can't be null"); } saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); } - return Futures.allAsList(futures); + return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); } @Override @@ -135,12 +151,12 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.allAsList(futures); } - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey(), ttl)); - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); } @@ -216,7 +232,7 @@ public class BaseTimeseriesService implements TimeseriesService { } else if (query.getAggregation() == null) { throw new IncorrectParameterException("Incorrect ReadTsKvQuery. Aggregation can't be empty"); } - if(!Aggregation.NONE.equals(query.getAggregation())) { + if (!Aggregation.NONE.equals(query.getAggregation())) { long step = Math.max(query.getInterval(), 1000); long intervalCounts = (query.getEndTs() - query.getStartTs()) / step; if (intervalCounts > maxTsIntervals || intervalCounts < 0) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 2f3fe4854c..b9d8c62833 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -60,6 +60,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -74,6 +75,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD protected static final int MIN_AGGREGATION_STEP_MS = 1000; public static final String ASC_ORDER = "ASC"; + public static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); + protected static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); @Autowired @@ -141,9 +144,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { List> futures = new ArrayList<>(); ttl = computeTtl(ttl); + int dataPointDays = tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); long partition = toPartitionTs(tsKvEntry.getTs()); DataType type = tsKvEntry.getDataType(); if (setNullValuesEnabled) { @@ -161,11 +165,11 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } BoundStatement stmt = stmtBuilder.build(); futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); - return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); + return Futures.transform(Futures.allAsList(futures), result -> dataPointDays, MoreExecutors.directExecutor()); } @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { if (isFixedPartitioning()) { return Futures.immediateFuture(null); } @@ -181,7 +185,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD stmtBuilder.setInt(4, (int) ttl); } BoundStatement stmt = stmtBuilder.build(); - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> 0); } @Override @@ -649,9 +653,10 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } /** - // * Select existing partitions from the table - // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity - // */ + * // * Select existing partitions from the table + * // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity + * // + */ private TbResultSetFuture fetchPartitions(TenantId tenantId, EntityId entityId, String key, long minPartition, long maxPartition) { Select select = QueryBuilder.selectFrom(ModelConstants.TS_KV_PARTITIONS_CF).column(ModelConstants.PARTITION_COLUMN) .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index dda2b227c3..2dd52580b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -31,9 +31,9 @@ public interface TimeseriesDao { ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); - ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); + ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 26184a5824..e1b494e2e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index d04d18772d..ff87e17d56 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -793,7 +793,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } } - List>> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -1272,7 +1272,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture> saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value);