diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 1cc4146ed7..7d80b115d5 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -28,7 +28,9 @@ import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; +import org.thingsboard.server.service.install.TsLatestDatabaseSchemaService; import org.thingsboard.server.service.install.migrate.EntitiesMigrateService; +import org.thingsboard.server.service.install.migrate.TsLatestMigrateService; import org.thingsboard.server.service.install.update.DataUpdateService; @Service @@ -51,6 +53,9 @@ public class ThingsboardInstallService { @Autowired private TsDatabaseSchemaService tsDatabaseSchemaService; + @Autowired(required = false) + private TsLatestDatabaseSchemaService tsLatestDatabaseSchemaService; + @Autowired private DatabaseEntitiesUpgradeService databaseEntitiesUpgradeService; @@ -72,6 +77,9 @@ public class ThingsboardInstallService { @Autowired(required = false) private EntitiesMigrateService entitiesMigrateService; + @Autowired(required = false) + private TsLatestMigrateService latestMigrateService; + public void performInstall() { try { if (isUpgrade) { @@ -82,6 +90,10 @@ public class ThingsboardInstallService { entitiesMigrateService.migrate(); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); + } else if ("3.0.1-cassandra".equals(upgradeFromVersion)) { + log.info("Migrating ThingsBoard latest timeseries data from cassandra to SQL database ..."); + latestMigrateService.migrate(); + log.info("Updating system data..."); } else { switch (upgradeFromVersion) { case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion @@ -181,6 +193,10 @@ public class ThingsboardInstallService { tsDatabaseSchemaService.createDatabaseSchema(); + if (tsLatestDatabaseSchemaService != null) { + tsLatestDatabaseSchemaService.createDatabaseSchema(); + } + log.info("Loading system data..."); componentDiscoveryService.discoverComponents(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsLatestDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsLatestDatabaseSchemaService.java new file mode 100644 index 0000000000..f2df8a50da --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsLatestDatabaseSchemaService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.NoSqlTsLatestDao; + +@Service +@NoSqlTsLatestDao +@Profile("install") +public class CassandraTsLatestDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService + implements TsLatestDatabaseSchemaService { + public CassandraTsLatestDatabaseSchemaService() { + super("schema-ts-latest.cql"); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java index 0b3232903b..8124356bba 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java @@ -18,11 +18,9 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @Service @HsqlDao -@SqlDao @Profile("install") public class HsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements EntityDatabaseSchemaService { diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java index 11da8b306a..364457f11e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java @@ -18,10 +18,8 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @Service -@SqlDao @PsqlDao @Profile("install") public class PsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index e244c307ff..e21472ee12 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.dashboard.DashboardService; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.service.install.sql.SqlDbHelper; import java.nio.charset.Charset; @@ -58,7 +57,6 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; @Service @Profile("install") @Slf4j -@SqlDao public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { private static final String SCHEMA_UPDATE_SQL = "schema_update.sql"; diff --git a/application/src/main/java/org/thingsboard/server/service/install/TsLatestDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/TsLatestDatabaseSchemaService.java new file mode 100644 index 0000000000..070cc2060e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/TsLatestDatabaseSchemaService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +public interface TsLatestDatabaseSchemaService extends DatabaseSchemaService { +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java index a4b4336d61..77459604ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.dao.cassandra.CassandraCluster; import org.thingsboard.server.dao.util.NoSqlAnyDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import java.sql.Connection; @@ -42,7 +41,6 @@ import static org.thingsboard.server.service.install.migrate.CassandraToSqlColum @Service @Profile("install") -@SqlDao @NoSqlAnyDao @Slf4j public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateService { diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java index dd67a6b055..b18f48ec85 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java @@ -128,7 +128,7 @@ public class CassandraToSqlTable { return this.validateColumnData(data); } - private CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { + protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { for (int i=0;i batchData, Connection conn) throws SQLException { + protected void batchInsert(List batchData, Connection conn) throws SQLException { boolean retry = false; for (CassandraToSqlColumnData[] data : batchData) { for (CassandraToSqlColumn column: this.columns) { @@ -269,7 +269,7 @@ public class CassandraToSqlTable { return Optional.empty(); } - private Statement createCassandraSelectStatement() { + protected Statement createCassandraSelectStatement() { StringBuilder selectStatementBuilder = new StringBuilder(); selectStatementBuilder.append("SELECT "); for (CassandraToSqlColumn column : columns) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java new file mode 100644 index 0000000000..ab22e3748c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java @@ -0,0 +1,205 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install.migrate; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.hibernate.exception.ConstraintViolationException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.UUIDConverter; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; +import org.thingsboard.server.dao.util.NoSqlAnyDao; +import org.thingsboard.server.service.install.EntityDatabaseSchemaService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.bigintColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.booleanColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.doubleColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.idColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.jsonColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.stringColumn; + +@Service +@Profile("install") +@NoSqlAnyDao +@Slf4j +public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateService { + + @Autowired + private EntityDatabaseSchemaService entityDatabaseSchemaService; + + @Autowired + private InsertLatestTsRepository insertLatestTsRepository; + + @Autowired + protected CassandraCluster cluster; + + @Autowired + protected TsKvDictionaryRepository dictionaryRepository; + + @Value("${spring.datasource.url}") + protected String dbUrl; + + @Value("${spring.datasource.username}") + protected String dbUserName; + + @Value("${spring.datasource.password}") + protected String dbPassword; + + private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); + + protected static final ReentrantLock tsCreationLock = new ReentrantLock(); + + @Override + public void migrate() throws Exception { + log.info("Performing migration of latest timeseries data from cassandra to SQL database ..."); + entityDatabaseSchemaService.createDatabaseSchema(false); + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + conn.setAutoCommit(false); + for (CassandraToSqlTable table : tables) { + table.migrateToSql(cluster.getSession(), conn); + } + } catch (Exception e) { + log.error("Unexpected error during ThingsBoard entities data migration!", e); + throw e; + } + entityDatabaseSchemaService.createDatabaseIndexes(); + } + + private List tables = Arrays.asList( + new CassandraToSqlTable("ts_kv_latest_cf", + idColumn("entity_id"), + stringColumn("key"), + bigintColumn("ts"), + booleanColumn("bool_v"), + stringColumn("str_v"), + bigintColumn("long_v"), + doubleColumn("dbl_v"), + jsonColumn("json_v")) { + + @Override + protected void batchInsert(List batchData, Connection conn) { + insertLatestTsRepository + .saveOrUpdate(batchData.stream().map(data -> getTsKvLatestEntity(data)).collect(Collectors.toList())); + } + + @Override + protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { + return data; + } + }); + + private TsKvLatestEntity getTsKvLatestEntity(CassandraToSqlColumnData[] data) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityId(UUIDConverter.fromString(data[0].getValue())); + latestEntity.setKey(getOrSaveKeyId(data[1].getValue())); + latestEntity.setTs(Long.parseLong(data[2].getValue())); + + String strV = data[4].getValue(); + if (strV != null) { + latestEntity.setStrValue(strV); + } else { + Long longV = null; + try { + longV = Long.parseLong(data[5].getValue()); + } catch (Exception e) { + } + if (longV != null) { + latestEntity.setLongValue(longV); + } else { + Double doubleV = null; + try { + doubleV = Double.parseDouble(data[6].getValue()); + } catch (Exception e) { + } + if (doubleV != null) { + latestEntity.setDoubleValue(doubleV); + } else { + + String jsonV = data[7].getValue(); + if (StringUtils.isNoneEmpty(jsonV)) { + latestEntity.setJsonValue(jsonV); + } else { + Boolean boolV = null; + try { + boolV = Boolean.parseBoolean(data[3].getValue()); + } catch (Exception e) { + } + if (boolV != null) { + latestEntity.setBooleanValue(boolV); + } else { + log.warn("All values in key-value row are nullable "); + } + } + } + } + } + return latestEntity; + } + + protected Integer getOrSaveKeyId(String strKey) { + Integer keyId = tsKvDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + tsCreationLock.lock(); + try { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + TsKvDictionary tsKvDictionary = new TsKvDictionary(); + tsKvDictionary.setKey(strKey); + try { + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (ConstraintViolationException e) { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + tsCreationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + tsKvDictionaryMap.put(strKey, keyId); + } + } + return keyId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/TsLatestMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/TsLatestMigrateService.java new file mode 100644 index 0000000000..491cac2c99 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/TsLatestMigrateService.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install.migrate; + +public interface TsLatestMigrateService { + + void migrate() throws Exception; +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 5fe6e0af1a..6e1eca4a7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.util.mapping.JacksonUtil; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; @@ -61,7 +60,6 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index 5b094c5c0e..a608ca257b 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.service.ttl.AbstractCleanUpService; import java.sql.Connection; @@ -28,7 +27,6 @@ import java.sql.DriverManager; import java.sql.SQLException; @PsqlDao -@SqlDao @Slf4j @Service public class EventsCleanUpService extends AbstractCleanUpService { diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index d6b08195b2..fb09a7eab4 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -19,11 +19,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.util.PsqlTsDao; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; import java.sql.Connection; -@PsqlTsDao +@SqlTsDao +@PsqlDao @Service @Slf4j public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService { @@ -33,9 +35,9 @@ public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpServi @Override protected void doCleanUp(Connection connection) { - long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);"); - log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved); - long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);"); - log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); + long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);"); + log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved); + long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);"); + log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); } } \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a482f4a219..bc6eebcf23 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -178,8 +178,8 @@ database: ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by single API call to fetch telemetry records ts: type: "${DATABASE_TS_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) - latest_ts: - type: "${DATABASE_TS_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) + ts_latest: + type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) # note: timescale works only with postgreSQL database for DATABASE_ENTITIES_TYPE. diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index ad81884f1f..a062fb7ebe 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -41,7 +41,9 @@ public class MqttNoSqlTestSuite { public static CustomCassandraCQLUnit cassandraUnit = new CustomCassandraCQLUnit( Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false)), + new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), + new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) + ), "cassandra-test.yaml", 30000l); @BeforeClass diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index bcf1222988..e267e5288d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") +@ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.ts_latest.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsLatestDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsLatestDao.java new file mode 100644 index 0000000000..b9e83ddb56 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsLatestDao.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnProperty(prefix = "database.ts_latest", value = "type", havingValue = "cassandra") +public @interface NoSqlTsLatestDao { +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java similarity index 86% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java index a215a16b29..9cf597e291 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java @@ -21,7 +21,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + +@ConditionalOnExpression("('${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") -public @interface PsqlTsAnyDao { +public @interface PsqlTsLatestAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDao.java similarity index 83% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDao.java index 95ca8c2b05..424e75ccdd 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDao.java @@ -21,5 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${database.ts.type}'=='sql' && '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") -public @interface PsqlTsDao { } \ No newline at end of file +@ConditionalOnExpression("'${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale'") +public @interface SqlTsLatestAnyDao { +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestDao.java similarity index 79% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestDao.java index 76db7920a6..a2c07a43eb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestDao.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.dao.util; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -public @interface SqlDao { +@ConditionalOnProperty(prefix = "database.ts_latest", value = "type", havingValue = "sql") +public @interface SqlTsLatestDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsOrTsLatestAnyDao.java similarity index 85% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsOrTsLatestAnyDao.java index 9be3321988..f2f5ecae4a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsOrTsLatestAnyDao.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") -public @interface SqlTsAnyDao { +@ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale' || '${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale'") +public @interface SqlTsOrTsLatestAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsLatestDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsLatestDao.java new file mode 100644 index 0000000000..579cd1a600 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsLatestDao.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnProperty(prefix = "database.ts_latest", value = "type", havingValue = "timescale") +public @interface TimescaleDBTsLatestDao { +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsOrTsLatestDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsOrTsLatestDao.java new file mode 100644 index 0000000000..dcc44a0014 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsOrTsLatestDao.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${database.ts.type}'=='timescale' || '${database.ts_latest.type}'=='timescale'") +public @interface TimescaleDBTsOrTsLatestDao { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java index e2519191e9..3a54545952 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java @@ -27,8 +27,8 @@ import org.thingsboard.server.dao.util.SqlTsDao; @Configuration @EnableAutoConfiguration @ComponentScan({"org.thingsboard.server.dao.sqlts.hsql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.hsql", "org.thingsboard.server.dao.sqlts.insert.latest.hsql", "org.thingsboard.server.dao.sqlts.latest", "org.thingsboard.server.dao.sqlts.dictionary"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts", "org.thingsboard.server.dao.model.sqlts.latest", "org.thingsboard.server.dao.model.sqlts.dictionary"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.hsql"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts"}) @EnableTransactionManagement @SqlTsDao @HsqlDao diff --git a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java new file mode 100644 index 0000000000..c814baebfa --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.HsqlDao; +import org.thingsboard.server.dao.util.SqlTsLatestDao; + +@Configuration +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.hsql"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.hsql", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableTransactionManagement +@SqlTsLatestDao +@HsqlDao +public class HsqlTsLatestDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java index 796f98d238..32db43271c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java @@ -21,7 +21,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.SqlDao; /** * @author Valerii Sosliuk @@ -32,7 +31,6 @@ import org.thingsboard.server.dao.util.SqlDao; @EnableJpaRepositories("org.thingsboard.server.dao.sql") @EntityScan("org.thingsboard.server.dao.model.sql") @EnableTransactionManagement -@SqlDao public class JpaDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java index 65f17709ca..363b53713f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java @@ -27,11 +27,11 @@ import org.thingsboard.server.dao.util.SqlTsDao; @Configuration @EnableAutoConfiguration @ComponentScan({"org.thingsboard.server.dao.sqlts.psql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.psql", "org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest", "org.thingsboard.server.dao.sqlts.dictionary"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts", "org.thingsboard.server.dao.model.sqlts.latest", "org.thingsboard.server.dao.model.sqlts.dictionary"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.psql"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts"}) @EnableTransactionManagement -@SqlTsDao @PsqlDao +@SqlTsDao public class PsqlTsDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java new file mode 100644 index 0000000000..338bd71e73 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsLatestDao; + +@Configuration +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.psql"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableTransactionManagement +@SqlTsLatestDao +@PsqlDao +public class PsqlTsLatestDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java new file mode 100644 index 0000000000..6b38df8284 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; + +@Configuration +@EnableAutoConfiguration +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.dictionary"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.dictionary"}) +@EnableTransactionManagement +@SqlTsOrTsLatestAnyDao +public class SqlTimeseriesDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java index 19ae98c736..673ff314c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java @@ -27,8 +27,8 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; @Configuration @EnableAutoConfiguration @ComponentScan({"org.thingsboard.server.dao.sqlts.timescale"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.insert.timescale", "org.thingsboard.server.dao.sqlts.dictionary", "org.thingsboard.server.dao.sqlts.latest"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale", "org.thingsboard.server.dao.model.sqlts.dictionary", "org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.insert.timescale"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale"}) @EnableTransactionManagement @TimescaleDBTsDao @PsqlDao diff --git a/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java new file mode 100644 index 0000000000..9765dd6d11 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.TimescaleDBTsLatestDao; + +@Configuration +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.timescale"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableTransactionManagement +@TimescaleDBTsLatestDao +@PsqlDao +public class TimescaleTsLatestDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java index ef9d287299..ace621cd6a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java @@ -18,10 +18,8 @@ package org.thingsboard.server.dao.sql; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.server.dao.util.SqlDao; @Component -@SqlDao public class JpaExecutorService extends AbstractListeningExecutor { @Value("${spring.datasource.hikari.maximumPoolSize}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 3c85cf2c60..1f82329c90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -21,25 +21,15 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.alarm.AlarmStatus; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.query.AlarmData; -import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.dao.model.sql.AlarmEntity; import org.thingsboard.server.dao.model.sql.AlarmInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; -import java.util.Collection; import java.util.List; -import java.util.Set; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -@SqlDao public interface AlarmRepository extends CrudRepository { @Query("SELECT a FROM AlarmEntity a WHERE a.originatorId = :originatorId AND a.type = :alarmType ORDER BY a.startTs DESC") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 2b42f81e6e..202281288b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -39,7 +39,6 @@ import org.thingsboard.server.dao.model.sql.AlarmEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.query.AlarmQueryRepository; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collection; @@ -54,7 +53,6 @@ import java.util.UUID; */ @Slf4j @Component -@SqlDao public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 6d87d75366..5e9d942720 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -@SqlDao public interface AssetRepository extends PagingAndSortingRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo) " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index e836aa1ef2..1e67802873 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collections; @@ -44,7 +43,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/19/2017. */ @Component -@SqlDao public class JpaAssetDao extends JpaAbstractSearchTextDao implements AssetDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 70500d38a0..01bfaa99b0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -25,7 +25,6 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -35,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; -@SqlDao @Repository @Slf4j public abstract class AttributeKvInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index 1114fe5e66..f3ef44e6c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -23,12 +23,10 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; -@SqlDao public interface AttributeKvRepository extends CrudRepository { @Query("SELECT a FROM AttributeKvEntity a WHERE a.id.entityType = :entityType " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java index ce90a2d8cd..718b9e49dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java @@ -19,12 +19,10 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import java.sql.Types; import java.util.List; -@SqlDao @HsqlDao @Repository @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index dedec0bc07..2803dec8dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -34,7 +34,6 @@ import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; -import org.thingsboard.server.dao.util.SqlDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -46,7 +45,6 @@ import java.util.stream.Collectors; @Component @Slf4j -@SqlDao public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService implements AttributesDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java index 020e63cd36..076a472da1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java @@ -18,9 +18,7 @@ package org.thingsboard.server.dao.sql.attributes; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao @PsqlDao @Repository @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index 1ffce82735..714b6eebd4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -30,14 +30,12 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.audit.AuditLogDao; import org.thingsboard.server.dao.model.sql.AuditLogEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.Objects; import java.util.UUID; @Component -@SqlDao public class JpaAuditLogDao extends JpaAbstractDao implements AuditLogDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java index 83c2ad1e23..668b31c84a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java @@ -23,14 +23,12 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.plugin.ComponentScope; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface ComponentDescriptorRepository extends PagingAndSortingRepository { ComponentDescriptorEntity findByClazz(String clazz); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index 7477da2f74..05e28f637c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -18,11 +18,9 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.Query; -@SqlDao @HsqlDao @Repository public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index c128eda12f..d27eb7d3d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.component.ComponentDescriptorDao; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.Optional; @@ -41,7 +40,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaBaseComponentDescriptorDao extends JpaAbstractSearchTextDao implements ComponentDescriptorDao { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java index dfe63ec69a..44a1cf8312 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java @@ -18,9 +18,7 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao @PsqlDao @Repository public class PsqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 66977419fe..bf2a845ec0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.CustomerEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface CustomerRepository extends PagingAndSortingRepository { @Query("SELECT c FROM CustomerEntity c WHERE c.tenantId = :tenantId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index a3b585c596..14c025de42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -25,7 +25,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.model.sql.CustomerEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.Optional; @@ -35,7 +34,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaCustomerDao extends JpaAbstractSearchTextDao implements CustomerDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java index f2e21f06c5..b0eb8f4d7b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.DashboardInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DashboardInfoRepository extends PagingAndSortingRepository { @Query("SELECT di FROM DashboardInfoEntity di WHERE di.tenantId = :tenantId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 76b8723f2a..bc2dcf19f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -17,13 +17,11 @@ package org.thingsboard.server.dao.sql.dashboard; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DashboardRepository extends CrudRepository { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 017cf2e328..8d637f0cbe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaDashboardDao extends JpaAbstractSearchTextDao implements DashboardDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java index ac8fb2aabc..6fadc233d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.dashboard.DashboardInfoDao; import org.thingsboard.server.dao.model.sql.DashboardInfoEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @@ -37,7 +36,6 @@ import java.util.UUID; */ @Slf4j @Component -@SqlDao public class JpaDashboardInfoDao extends JpaAbstractSearchTextDao implements DashboardInfoDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index 8995a33b3a..c4577dd1e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -17,14 +17,12 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DeviceCredentialsRepository extends CrudRepository { DeviceCredentialsEntity findByDeviceId(UUID deviceId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index 4cc82b37cf..828a9e28f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DeviceRepository extends PagingAndSortingRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.DeviceInfoEntity(d, c.title, c.additionalInfo) " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 12a38d2642..a2cde30fd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -24,7 +24,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.device.DeviceCredentialsDao; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @@ -32,7 +31,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 03f7c84a06..e8e36a7fd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -32,7 +32,6 @@ import org.thingsboard.server.dao.device.DeviceDao; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collections; @@ -45,7 +44,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaDeviceDao extends JpaAbstractSearchTextDao implements DeviceDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index d119648d76..0c7c55426e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/31/2017. */ -@SqlDao public interface EntityViewRepository extends PagingAndSortingRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.EntityViewInfoEntity(e, c.title, c.additionalInfo) " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 556d8dab5e..0e42b579fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collections; @@ -44,7 +43,6 @@ import java.util.UUID; * Created by Victor Basanets on 8/31/2017. */ @Component -@SqlDao public class JpaEntityViewDao extends JpaAbstractSearchTextDao implements EntityViewDao { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java index 6a09a70ebc..5696bfbf50 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.dao.model.sql.EventEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/3/2017. */ -@SqlDao public interface EventRepository extends PagingAndSortingRepository { EventEntity findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(UUID tenantId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index 1122499721..722c370ac9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -18,11 +18,9 @@ package org.thingsboard.server.dao.sql.event; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.Query; -@SqlDao @HsqlDao @Repository public class HsqlEventInsertRepository extends AbstractEventInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index c9e9750a5f..20f49ac8d8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; -import org.springframework.data.jpa.domain.Specification; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Event; @@ -34,10 +33,7 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; -import javax.persistence.criteria.Predicate; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -50,7 +46,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Slf4j @Component -@SqlDao public class JpaBaseEventDao extends JpaAbstractDao implements EventDao { private final UUID systemTenantId = NULL_UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java index e0e3e728ea..1f401e5eb2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java @@ -19,10 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @Slf4j -@SqlDao @PsqlDao @Repository public class PsqlEventInsertRepository extends AbstractEventInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index f33667271b..29b070c9e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Arrays; @@ -52,7 +51,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -@SqlDao @Repository @Slf4j public class DefaultAlarmQueryRepository implements AlarmQueryRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 06a6f7f059..7e088f2016 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -50,7 +50,6 @@ import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.EntityTypeFilter; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Arrays; import java.util.Collections; @@ -61,7 +60,6 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; -@SqlDao @Repository @Slf4j public class DefaultEntityQueryRepository implements EntityQueryRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java index 7e8e77e25f..2c77c5d833 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java @@ -24,10 +24,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.dao.entity.EntityQueryDao; -import org.thingsboard.server.dao.util.SqlDao; @Component -@SqlDao public class JpaEntityQueryDao implements EntityQueryDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java index fc262945e1..e017831607 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java @@ -20,12 +20,10 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.Query; @HsqlDao -@SqlDao @Repository @Transactional public class HsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b96d3db110..3b6e7a83cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -30,7 +30,6 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.criteria.Predicate; import java.util.ArrayList; @@ -41,7 +40,6 @@ import java.util.List; */ @Slf4j @Component -@SqlDao public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService implements RelationDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java index dbef233811..7325fc0fbc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java @@ -19,10 +19,8 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @PsqlDao -@SqlDao @Repository @Transactional public class PsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 95c67e0b9f..ada7a44565 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -20,12 +20,10 @@ import org.springframework.data.repository.CrudRepository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; -@SqlDao public interface RelationRepository extends CrudRepository, JpaSpecificationExecutor { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 63e2889bc5..57d1545a94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -26,14 +26,12 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.RuleChainEntity; import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @Slf4j @Component -@SqlDao public class JpaRuleChainDao extends JpaAbstractSearchTextDao implements RuleChainDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index e29a988fc1..e5302e2618 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import org.thingsboard.server.dao.rule.RuleNodeDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; @Slf4j @Component -@SqlDao public class JpaRuleNodeDao extends JpaAbstractSearchTextDao implements RuleNodeDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 24b795cae4..228d4d26f4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -21,11 +21,9 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.RuleChainEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; -@SqlDao public interface RuleChainRepository extends PagingAndSortingRepository { @Query("SELECT rc FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index d3f51ec4f6..58982f6f73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -17,9 +17,7 @@ package org.thingsboard.server.dao.sql.rule; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao public interface RuleNodeRepository extends CrudRepository { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index ad56b5754c..4e1d44d7f2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -25,13 +25,11 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @Component @Slf4j -@SqlDao public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index 2c1dcb7516..6f22af118c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -26,7 +26,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @@ -36,7 +35,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 4/30/2017. */ @Component -@SqlDao public class JpaTenantDao extends JpaAbstractSearchTextDao implements TenantDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 04471d2e26..012920e187 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.TenantEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 4/30/2017. */ -@SqlDao public interface TenantRepository extends PagingAndSortingRepository { @Query("SELECT t FROM TenantEntity t WHERE t.region = :region " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java index 5c732bafd8..87e5a496d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java @@ -24,7 +24,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserCredentialsDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @@ -32,7 +31,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 4/22/2017. */ @Component -@SqlDao public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index b0805c1a52..42092f7ebe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.UserEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.user.UserDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @@ -38,7 +37,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; * @author Valerii Sosliuk */ @Component -@SqlDao public class JpaUserDao extends JpaAbstractSearchTextDao implements UserDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java index 57e954f796..4b1791ddcb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java @@ -17,14 +17,12 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 4/22/2017. */ -@SqlDao public interface UserCredentialsRepository extends CrudRepository { UserCredentialsEntity findByUserId(UUID userId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 5bae8fbe68..1d512a7944 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -22,14 +22,12 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.sql.UserEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * @author Valerii Sosliuk */ -@SqlDao public interface UserRepository extends PagingAndSortingRepository { UserEntity findByEmail(String email); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index 4be5e5f514..805897caf4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.WidgetTypeEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.widget.WidgetTypeDao; import java.util.List; @@ -32,7 +31,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 4/29/2017. */ @Component -@SqlDao public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 8e84f95627..e0611a6263 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; import java.util.Objects; @@ -37,7 +36,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; * Created by Valerii Sosliuk on 4/23/2017. */ @Component -@SqlDao public class JpaWidgetsBundleDao extends JpaAbstractSearchTextDao implements WidgetsBundleDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java index 387f21d7d7..6167e64234 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.WidgetTypeEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -25,7 +24,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 4/29/2017. */ -@SqlDao public interface WidgetTypeRepository extends CrudRepository { List findByTenantIdAndBundleAlias(UUID tenantId, String bundleAlias); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index f442a9739e..b4030f90de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 4/23/2017. */ -@SqlDao public interface WidgetsBundleRepository extends PagingAndSortingRepository { WidgetsBundleEntity findWidgetsBundleByTenantIdAndAlias(UUID tenantId, String alias); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 1d8194d805..079bf1b660 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sqlts; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; @@ -33,7 +32,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; @@ -64,7 +62,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @PostConstruct protected void init() { - super.init(); TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() .logName("TS") .batchSize(tsBatchSize) @@ -80,7 +77,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @PreDestroy protected void destroy() { - super.destroy(); if (tsQueue != null) { tsQueue.destroy(); } @@ -98,26 +94,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq }); } - @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - return getSaveLatestFuture(entityId, tsKvEntry); - } - - @Override - public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return getRemoveLatestFuture(entityId, query); - } - - @Override - public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - return getFindLatestFuture(entityId, key); - } - - @Override - public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - return getFindAllLatestFuture(entityId); - } - @Override public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { return Futures.immediateFuture(null); @@ -134,7 +110,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - protected ListenableFuture> findAllAsync(EntityId entityId, ReadTsKvQuery query) { + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(entityId, query); } else { @@ -151,8 +127,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } } - @Override - protected ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 9710edad4a..9ab8aa3dfc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -16,96 +16,28 @@ package org.thingsboard.server.dao.sqlts; import com.google.common.base.Function; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; -import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; -import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; -import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @Slf4j -public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { - - private static final String DESC_ORDER = "DESC"; - - private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); - - private static final ReentrantLock tsCreationLock = new ReentrantLock(); - - @Autowired - private TsKvLatestRepository tsKvLatestRepository; - - @Autowired - private SearchTsKvLatestRepository searchTsKvLatestRepository; - - @Autowired - private InsertLatestTsRepository insertLatestTsRepository; - - @Autowired - private TsKvDictionaryRepository dictionaryRepository; - - private TbSqlBlockingQueueWrapper tsLatestQueue; - - @Value("${sql.ts_latest.batch_size:1000}") - private int tsLatestBatchSize; - - @Value("${sql.ts_latest.batch_max_delay:100}") - private long tsLatestMaxDelay; - - @Value("${sql.ts_latest.stats_print_interval_ms:1000}") - private long tsLatestStatsPrintIntervalMs; - - @Value("${sql.ts_latest.batch_threads:4}") - private int tsLatestBatchThreads; +public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { @Autowired protected ScheduledLogExecutorComponent logExecutor; - @Autowired - private StatsFactory statsFactory; - @Value("${sql.ts.batch_size:1000}") protected int tsBatchSize; @@ -121,44 +53,10 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx @Value("${sql.timescale.batch_threads:4}") protected int timescaleBatchThreads; - @PostConstruct - protected void init() { - TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() - .logName("TS Latest") - .batchSize(tsLatestBatchSize) - .maxDelay(tsLatestMaxDelay) - .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) - .statsNamePrefix("ts.latest") - .build(); - - java.util.function.Function hashcodeFunction = entity -> entity.getEntityId().hashCode(); - tsLatestQueue = new TbSqlBlockingQueueWrapper<>(tsLatestParams, hashcodeFunction, tsLatestBatchThreads, statsFactory); - - tsLatestQueue.init(logExecutor, v -> { - Map trueLatest = new HashMap<>(); - v.forEach(ts -> { - TsKey key = new TsKey(ts.getEntityId(), ts.getKey()); - TsKvLatestEntity old = trueLatest.get(key); - if (old == null || old.getTs() < ts.getTs()) { - trueLatest.put(key, ts); - } - }); - List latestEntities = new ArrayList<>(trueLatest.values()); - insertLatestTsRepository.saveOrUpdate(latestEntities); - }); - } - - @PreDestroy - protected void destroy() { - if (tsLatestQueue != null) { - tsLatestQueue.destroy(); - } - } - protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries .stream() - .map(query -> findAllAsync(entityId, query)) + .map(query -> findAllAsync(tenantId, entityId, query)) .collect(Collectors.toList()); return Futures.transform(Futures.allAsList(futures), new Function>, List>() { @Nullable @@ -174,168 +72,4 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx } }, service); } - - protected abstract ListenableFuture> findAllAsync(EntityId entityId, ReadTsKvQuery query); - - protected abstract ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query); - - protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, service); - } - - protected ListenableFuture> findNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) { - long startTs = 0; - long endTs = query.getStartTs() - 1; - ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, - Aggregation.NONE, DESC_ORDER); - return findAllAsync(entityId, findNewLatestQuery); - } - - protected ListenableFuture getFindLatestFuture(EntityId entityId, String key) { - TsKvLatestCompositeKey compositeKey = - new TsKvLatestCompositeKey( - entityId.getId(), - getOrSaveKeyId(key)); - Optional entry = tsKvLatestRepository.findById(compositeKey); - TsKvEntry result; - if (entry.isPresent()) { - TsKvLatestEntity tsKvLatestEntity = entry.get(); - tsKvLatestEntity.setStrKey(key); - result = DaoUtil.getData(tsKvLatestEntity); - } else { - result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); - } - return Futures.immediateFuture(result); - } - - protected ListenableFuture getRemoveLatestFuture(EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture latestFuture = getFindLatestFuture(entityId, query.getKey()); - - ListenableFuture booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { - long ts = tsKvEntry.getTs(); - return ts > query.getStartTs() && ts <= query.getEndTs(); - }, service); - - ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); - latestEntity.setEntityId(entityId.getId()); - latestEntity.setKey(getOrSaveKeyId(query.getKey())); - return service.submit(() -> { - tsKvLatestRepository.delete(latestEntity); - return null; - }); - } - return Futures.immediateFuture(null); - }, service); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - Futures.addCallback(removedLatestFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - if (query.getRewriteLatestIfDeleted()) { - ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - return getNewLatestEntryFuture(entityId, query); - } - return Futures.immediateFuture(null); - }, service); - - try { - resultFuture.set(savedLatestFuture.get()); - } catch (InterruptedException | ExecutionException e) { - log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); - } - } else { - resultFuture.set(null); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to process remove of the latest value", entityId, t); - } - }, MoreExecutors.directExecutor()); - return resultFuture; - } - - protected ListenableFuture> getFindAllLatestFuture(EntityId entityId) { - return Futures.immediateFuture( - DaoUtil.convertDataList(Lists.newArrayList( - searchTsKvLatestRepository.findAllByEntityId(entityId.getId())))); - } - - protected ListenableFuture getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); - latestEntity.setEntityId(entityId.getId()); - latestEntity.setTs(tsKvEntry.getTs()); - latestEntity.setKey(getOrSaveKeyId(tsKvEntry.getKey())); - latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); - latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); - latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); - latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - latestEntity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - - return tsLatestQueue.add(latestEntity); - } - - protected Integer getOrSaveKeyId(String strKey) { - Integer keyId = tsKvDictionaryMap.get(strKey); - if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - if (!tsKvDictionaryOptional.isPresent()) { - tsCreationLock.lock(); - try { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - if (!tsKvDictionaryOptional.isPresent()) { - TsKvDictionary tsKvDictionary = new TsKvDictionary(); - tsKvDictionary.setKey(strKey); - try { - TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); - tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); - keyId = saved.getKeyId(); - } catch (ConstraintViolationException e) { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); - tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); - keyId = dictionary.getKeyId(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - } - } finally { - tsCreationLock.unlock(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - tsKvDictionaryMap.put(strKey, keyId); - } - } - return keyId; - } - - private ListenableFuture getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture> future = findNewLatestEntryFuture(entityId, query); - return Futures.transformAsync(future, entryList -> { - if (entryList.size() == 1) { - return getSaveLatestFuture(entityId, entryList.get(0)); - } else { - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, service); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..345dc26e48 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface AggregationTimeseriesDao { + + ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java new file mode 100644 index 0000000000..1d7fbbaac0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.hibernate.exception.ConstraintViolationException; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@Slf4j +public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { + + private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); + + protected static final ReentrantLock tsCreationLock = new ReentrantLock(); + + @Autowired + protected TsKvDictionaryRepository dictionaryRepository; + + protected Integer getOrSaveKeyId(String strKey) { + Integer keyId = tsKvDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + tsCreationLock.lock(); + try { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + TsKvDictionary tsKvDictionary = new TsKvDictionary(); + tsKvDictionary.setKey(strKey); + try { + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (ConstraintViolationException e) { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + tsCreationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + tsKvDictionaryMap.put(strKey, keyId); + } + } + return keyId; + } + + protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { + return Futures.transform(future, new Function>, List>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, service); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java new file mode 100644 index 0000000000..ee7c02d558 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -0,0 +1,261 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; +import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; +import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; +import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; +import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; + +import javax.annotation.Nullable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +@Slf4j +@Component +@SqlTsLatestAnyDao +public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao implements TimeseriesLatestDao { + + private static final String DESC_ORDER = "DESC"; + + @Autowired + private TsKvLatestRepository tsKvLatestRepository; + + @Autowired + protected AggregationTimeseriesDao aggregationTimeseriesDao; + + @Autowired + private SearchTsKvLatestRepository searchTsKvLatestRepository; + + @Autowired + private InsertLatestTsRepository insertLatestTsRepository; + + private TbSqlBlockingQueueWrapper tsLatestQueue; + + @Value("${sql.ts_latest.batch_size:1000}") + private int tsLatestBatchSize; + + @Value("${sql.ts_latest.batch_max_delay:100}") + private long tsLatestMaxDelay; + + @Value("${sql.ts_latest.stats_print_interval_ms:1000}") + private long tsLatestStatsPrintIntervalMs; + + @Value("${sql.ts_latest.batch_threads:4}") + private int tsLatestBatchThreads; + + @Autowired + protected ScheduledLogExecutorComponent logExecutor; + + @Autowired + private StatsFactory statsFactory; + + @PostConstruct + protected void init() { + TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() + .logName("TS Latest") + .batchSize(tsLatestBatchSize) + .maxDelay(tsLatestMaxDelay) + .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) + .statsNamePrefix("ts.latest") + .build(); + + java.util.function.Function hashcodeFunction = entity -> entity.getEntityId().hashCode(); + tsLatestQueue = new TbSqlBlockingQueueWrapper<>(tsLatestParams, hashcodeFunction, tsLatestBatchThreads, statsFactory); + + tsLatestQueue.init(logExecutor, v -> { + Map trueLatest = new HashMap<>(); + v.forEach(ts -> { + TsKey key = new TsKey(ts.getEntityId(), ts.getKey()); + TsKvLatestEntity old = trueLatest.get(key); + if (old == null || old.getTs() < ts.getTs()) { + trueLatest.put(key, ts); + } + }); + List latestEntities = new ArrayList<>(trueLatest.values()); + insertLatestTsRepository.saveOrUpdate(latestEntities); + }); + } + + @PreDestroy + protected void destroy() { + if (tsLatestQueue != null) { + tsLatestQueue.destroy(); + } + } + + @Override + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + return getSaveLatestFuture(entityId, tsKvEntry); + } + + @Override + public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return getRemoveLatestFuture(tenantId, entityId, query); + } + + @Override + public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { + return getFindLatestFuture(entityId, key); + } + + @Override + public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { + return getFindAllLatestFuture(entityId); + } + + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); + return Futures.transformAsync(future, entryList -> { + if (entryList.size() == 1) { + return getSaveLatestFuture(entityId, entryList.get(0)); + } else { + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); + } + return Futures.immediateFuture(null); + }, service); + } + + private ListenableFuture> findNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long startTs = 0; + long endTs = query.getStartTs() - 1; + ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, + Aggregation.NONE, DESC_ORDER); + return aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery); + } + + protected ListenableFuture getFindLatestFuture(EntityId entityId, String key) { + TsKvLatestCompositeKey compositeKey = + new TsKvLatestCompositeKey( + entityId.getId(), + getOrSaveKeyId(key)); + Optional entry = tsKvLatestRepository.findById(compositeKey); + TsKvEntry result; + if (entry.isPresent()) { + TsKvLatestEntity tsKvLatestEntity = entry.get(); + tsKvLatestEntity.setStrKey(key); + result = DaoUtil.getData(tsKvLatestEntity); + } else { + result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); + } + return Futures.immediateFuture(result); + } + + protected ListenableFuture getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture latestFuture = getFindLatestFuture(entityId, query.getKey()); + + ListenableFuture booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { + long ts = tsKvEntry.getTs(); + return ts > query.getStartTs() && ts <= query.getEndTs(); + }, service); + + ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityId(entityId.getId()); + latestEntity.setKey(getOrSaveKeyId(query.getKey())); + return service.submit(() -> { + tsKvLatestRepository.delete(latestEntity); + return null; + }); + } + return Futures.immediateFuture(null); + }, service); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + Futures.addCallback(removedLatestFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + if (query.getRewriteLatestIfDeleted()) { + ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + return getNewLatestEntryFuture(tenantId, entityId, query); + } + return Futures.immediateFuture(null); + }, service); + + try { + resultFuture.set(savedLatestFuture.get()); + } catch (InterruptedException | ExecutionException e) { + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); + } + } else { + resultFuture.set(null); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to process remove of the latest value", entityId, t); + } + }, MoreExecutors.directExecutor()); + return resultFuture; + } + + protected ListenableFuture> getFindAllLatestFuture(EntityId entityId) { + return Futures.immediateFuture( + DaoUtil.convertDataList(Lists.newArrayList( + searchTsKvLatestRepository.findAllByEntityId(entityId.getId())))); + } + + protected ListenableFuture getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityId(entityId.getId()); + latestEntity.setTs(tsKvEntry.getTs()); + latestEntity.setKey(getOrSaveKeyId(tsKvEntry.getKey())); + latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); + latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); + latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); + latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + latestEntity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); + + return tsLatestQueue.add(latestEntity); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java index 55d2d031d9..76872ab176 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java @@ -18,11 +18,11 @@ package org.thingsboard.server.dao.sqlts.dictionary; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; -import org.thingsboard.server.dao.util.SqlTsAnyDao; +import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; import java.util.Optional; -@SqlTsAnyDao +@SqlTsOrTsLatestAnyDao public interface TsKvDictionaryRepository extends CrudRepository { Optional findByKeyId(int keyId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java index 224dc52805..2b6c2b9012 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java @@ -22,14 +22,14 @@ import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; +import org.thingsboard.server.dao.util.SqlTsLatestDao; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; import java.util.List; -@SqlTsDao +@SqlTsLatestDao @HsqlDao @Repository @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java index 41abae52f8..034ef26c00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java @@ -23,7 +23,7 @@ import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.dao.util.PsqlTsLatestAnyDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -32,7 +32,7 @@ import java.util.ArrayList; import java.util.List; -@PsqlTsAnyDao +@PsqlTsLatestAnyDao @Repository @Transactional public class PsqlLatestInsertTsRepository extends AbstractInsertRepository implements InsertLatestTsRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java index b5da093b6f..19687a94ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java @@ -17,14 +17,14 @@ package org.thingsboard.server.dao.sqlts.latest; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.util.SqlTsAnyDao; +import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import java.util.List; import java.util.UUID; -@SqlTsAnyDao +@SqlTsLatestAnyDao @Repository public class SearchTsKvLatestRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 5232b88489..bb53106254 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -18,9 +18,7 @@ package org.thingsboard.server.dao.sqlts.latest; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao public interface TsKvLatestRepository extends CrudRepository { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java index 7d0aefb62c..8c50bcb38c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java @@ -41,11 +41,10 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; - @Component @Slf4j -@SqlTsDao @PsqlDao +@SqlTsDao public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao { private final Map partitions = new ConcurrentHashMap<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java index 28b666b03b..e12a0219d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.sqlts.timescale; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; -import org.thingsboard.server.dao.util.TimescaleDBTsDao; +import org.thingsboard.server.dao.util.TimescaleDBTsOrTsLatestDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -27,7 +27,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; @Repository -@TimescaleDBTsDao +@TimescaleDBTsOrTsLatestDao public class AggregationRepository { public static final String FIND_AVG = "findAvg"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 738c98d3b7..71647f88f9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -72,7 +72,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @PostConstruct protected void init() { - super.init(); TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() .logName("TS Timescale") .batchSize(tsBatchSize) @@ -89,14 +88,60 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @PreDestroy protected void destroy() { - super.destroy(); if (tsQueue != null) { tsQueue.destroy(); } } @Override - protected ListenableFuture> findAllAsync(EntityId entityId, ReadTsKvQuery query) { + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + return processFindAllAsync(tenantId, entityId, queries); + } + + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + String strKey = tsKvEntry.getKey(); + Integer keyId = getOrSaveKeyId(strKey); + TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); + entity.setEntityId(entityId.getId()); + entity.setTs(tsKvEntry.getTs()); + entity.setKey(keyId); + entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); + entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); + + log.trace("Saving entity to timescale db: {}", entity); + return tsQueue.add(entity); + } + + @Override + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + return Futures.immediateFuture(null); + } + + @Override + public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + String strKey = query.getKey(); + Integer keyId = getOrSaveKeyId(strKey); + return service.submit(() -> { + tsKvRepository.delete( + entityId.getId(), + keyId, + query.getStartTs(), + query.getEndTs()); + return null; + }); + } + + @Override + public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return service.submit(() -> null); + } + + @Override + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(entityId, query); } else { @@ -108,8 +153,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } } - @Override - protected ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); List timescaleTsKvEntities = tsKvRepository.findAllWithLimit( @@ -153,73 +197,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements }, MoreExecutors.directExecutor()); } - @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { - return processFindAllAsync(tenantId, entityId, queries); - } - - @Override - public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - return getFindLatestFuture(entityId, key); - } - - @Override - public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - return getFindAllLatestFuture(entityId); - } - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - String strKey = tsKvEntry.getKey(); - Integer keyId = getOrSaveKeyId(strKey); - TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); - entity.setEntityId(entityId.getId()); - entity.setTs(tsKvEntry.getTs()); - entity.setKey(keyId); - entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); - entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); - entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); - entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - - log.trace("Saving entity to timescale db: {}", entity); - return tsQueue.add(entity); - } - - @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - return Futures.immediateFuture(null); - } - - @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - return getSaveLatestFuture(entityId, tsKvEntry); - } - - @Override - public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - String strKey = query.getKey(); - Integer keyId = getOrSaveKeyId(strKey); - return service.submit(() -> { - tsKvRepository.delete( - entityId.getId(), - keyId, - query.getStartTs(), - query.getEndTs()); - return null; - }); - } - - @Override - public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return getRemoveLatestFuture(entityId, query); - } - - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return service.submit(() -> null); - } - private CompletableFuture> switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId) { switch (aggregation) { case AVG: diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java index d4e80dc1f5..c6fb3574a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java @@ -23,12 +23,12 @@ import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvCompositeKey; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; -import org.thingsboard.server.dao.util.TimescaleDBTsDao; +import org.thingsboard.server.dao.util.TimescaleDBTsOrTsLatestDao; import java.util.List; import java.util.UUID; -@TimescaleDBTsDao +@TimescaleDBTsOrTsLatestDao public interface TsKvTimescaleRepository extends CrudRepository { @Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java index 5b446129a8..881de11fa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java @@ -24,13 +24,11 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.ts.TsKvCompositeKey; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; -@SqlDao public interface TsKvRepository extends CrudRepository { @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AbstractCassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AbstractCassandraBaseTimeseriesDao.java new file mode 100644 index 0000000000..3916a2fd10 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AbstractCassandraBaseTimeseriesDao.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.datastax.oss.driver.api.core.cql.Row; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public abstract class AbstractCassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao { + public static final String DESC_ORDER = "DESC"; + public static final String GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID = "Generated query [{}] for entityType {} and entityId {}"; + public static final String INSERT_INTO = "INSERT INTO "; + public static final String SELECT_PREFIX = "SELECT "; + public static final String EQUALS_PARAM = " = ? "; + + public static KvEntry toKvEntry(Row row, String key) { + KvEntry kvEntry = null; + String strV = row.get(ModelConstants.STRING_VALUE_COLUMN, String.class); + if (strV != null) { + kvEntry = new StringDataEntry(key, strV); + } else { + Long longV = row.get(ModelConstants.LONG_VALUE_COLUMN, Long.class); + if (longV != null) { + kvEntry = new LongDataEntry(key, longV); + } else { + Double doubleV = row.get(ModelConstants.DOUBLE_VALUE_COLUMN, Double.class); + if (doubleV != null) { + kvEntry = new DoubleDataEntry(key, doubleV); + } else { + Boolean boolV = row.get(ModelConstants.BOOLEAN_VALUE_COLUMN, Boolean.class); + if (boolV != null) { + kvEntry = new BooleanDataEntry(key, boolV); + } else { + String jsonV = row.get(ModelConstants.JSON_VALUE_COLUMN, String.class); + if (StringUtils.isNoneEmpty(jsonV)) { + kvEntry = new JsonDataEntry(key, jsonV); + } else { + log.warn("All values in key-value row are nullable "); + } + } + } + } + } + return kvEntry; + } + + protected List convertResultToTsKvEntryList(List rows) { + List entries = new ArrayList<>(rows.size()); + if (!rows.isEmpty()) { + rows.forEach(row -> entries.add(convertResultToTsKvEntry(row))); + } + return entries; + } + + private TsKvEntry convertResultToTsKvEntry(Row row) { + String key = row.getString(ModelConstants.KEY_COLUMN); + long ts = row.getLong(ModelConstants.TS_COLUMN); + return new BasicTsKvEntry(ts, toKvEntry(row, key)); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 376ea6499e..b78393b0a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -59,6 +59,9 @@ public class BaseTimeseriesService implements TimeseriesService { @Autowired private TimeseriesDao timeseriesDao; + @Autowired + private TimeseriesLatestDao timeseriesLatestDao; + @Autowired private EntityViewService entityViewService; @@ -103,7 +106,7 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.immediateFuture(new ArrayList<>()); } } - keys.forEach(key -> futures.add(timeseriesDao.findLatest(tenantId, entityId, key))); + keys.forEach(key -> futures.add(timeseriesLatestDao.findLatest(tenantId, entityId, key))); return Futures.allAsList(futures); } @@ -119,7 +122,7 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.immediateFuture(new ArrayList<>()); } } else { - return timeseriesDao.findAllLatest(tenantId, entityId); + return timeseriesLatestDao.findAllLatest(tenantId, entityId); } } @@ -151,7 +154,7 @@ public class BaseTimeseriesService implements TimeseriesService { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey(), ttl)); - futures.add(timeseriesDao.saveLatest(tenantId, entityId, tsKvEntry)); + futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); } @@ -187,7 +190,7 @@ public class BaseTimeseriesService implements TimeseriesService { private void deleteAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, DeleteTsKvQuery query) { futures.add(timeseriesDao.remove(tenantId, entityId, query)); - futures.add(timeseriesDao.removeLatest(tenantId, entityId, query)); + futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); futures.add(timeseriesDao.removePartition(tenantId, entityId, query)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 3eeff89a90..0dcef7c289 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -20,7 +20,6 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Row; -import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.datastax.oss.driver.api.querybuilder.select.Select; import com.google.common.base.Function; @@ -30,7 +29,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; @@ -39,21 +37,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -68,7 +60,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -79,16 +70,11 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @Component @Slf4j @NoSqlTsDao -public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implements TimeseriesDao { +public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesDao implements TimeseriesDao, AggregationTimeseriesDao { - private static final int MIN_AGGREGATION_STEP_MS = 1000; - public static final String INSERT_INTO = "INSERT INTO "; - public static final String GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID = "Generated query [{}] for entityType {} and entityId {}"; - public static final String SELECT_PREFIX = "SELECT "; - public static final String EQUALS_PARAM = " = ? "; + protected static final int MIN_AGGREGATION_STEP_MS = 1000; public static final String ASC_ORDER = "ASC"; - public static final String DESC_ORDER = "DESC"; - private static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); + protected static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); @Autowired private Environment environment; @@ -106,13 +92,10 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem private PreparedStatement partitionInsertStmt; private PreparedStatement partitionInsertTtlStmt; - private PreparedStatement latestInsertStmt; private PreparedStatement[] saveStmts; private PreparedStatement[] saveTtlStmts; private PreparedStatement[] fetchStmtsAsc; private PreparedStatement[] fetchStmtsDesc; - private PreparedStatement findLatestStmt; - private PreparedStatement findAllLatestStmt; private PreparedStatement deleteStmt; private PreparedStatement deletePartitionStmt; @@ -157,8 +140,113 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem }, readResultsProcessingExecutor); } + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + List> futures = new ArrayList<>(); + ttl = computeTtl(ttl); + long partition = toPartitionTs(tsKvEntry.getTs()); + DataType type = tsKvEntry.getDataType(); + if (setNullValuesEnabled) { + processSetNullValues(tenantId, entityId, tsKvEntry, ttl, futures, partition, type); + } + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getSaveStmt(type) : getSaveTtlStmt(type)).bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()) + .setUuid(1, entityId.getId()) + .setString(2, tsKvEntry.getKey()) + .setLong(3, partition) + .setLong(4, tsKvEntry.getTs()); + addValue(tsKvEntry, stmtBuilder, 5); + if (ttl > 0) { + stmtBuilder.setInt(6, (int) ttl); + } + BoundStatement stmt = stmtBuilder.build(); + futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); + return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + if (isFixedPartitioning()) { + return Futures.immediateFuture(null); + } + ttl = computeTtl(ttl); + long partition = toPartitionTs(tsKvEntryTs); + log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key); + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt()).bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()) + .setUuid(1, entityId.getId()) + .setLong(2, partition) + .setString(3, key); + if (ttl > 0) { + stmtBuilder.setInt(4, (int) ttl); + } + BoundStatement stmt = stmtBuilder.build(); + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); + } - private ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long minPartition = toPartitionTs(query.getStartTs()); + long maxPartition = toPartitionTs(query.getEndTs()); + + TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); + + Futures.addCallback(partitionsListFuture, new FutureCallback>() { + @Override + public void onSuccess(@Nullable List partitions) { + QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitions); + deleteAsync(tenantId, cursor, resultFuture); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); + } + }, readResultsProcessingExecutor); + return resultFuture; + } + + @Override + public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long minPartition = toPartitionTs(query.getStartTs()); + long maxPartition = toPartitionTs(query.getEndTs()); + if (minPartition == maxPartition) { + return Futures.immediateFuture(null); + } else { + TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); + + Futures.addCallback(partitionsListFuture, new FutureCallback>() { + @Override + public void onSuccess(@Nullable List partitions) { + int index = 0; + if (minPartition != query.getStartTs()) { + index = 1; + } + List partitionsToDelete = new ArrayList<>(); + for (int i = index; i < partitions.size() - 1; i++) { + partitionsToDelete.add(partitions.get(i)); + } + QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitionsToDelete); + deletePartitionAsync(tenantId, cursor, resultFuture); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); + } + }, readResultsProcessingExecutor); + return resultFuture; + } + } + + @Override + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { @@ -183,18 +271,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem } } - public boolean isFixedPartitioning() { - return tsFormat.getTruncateUnit().equals(ChronoUnit.FOREVER); - } - - private ListenableFuture> getPartitionsFuture(TenantId tenantId, ReadTsKvQuery query, EntityId entityId, long minPartition, long maxPartition) { - if (isFixedPartitioning()) { //no need to fetch partitions from DB - return Futures.immediateFuture(FIXED_PARTITION); - } - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); - return Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); - } - private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); @@ -287,10 +363,18 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem private AsyncFunction> getPartitionsArrayFunction() { return rs -> - Futures.transform(rs.allRows(readResultsProcessingExecutor), rows -> - rows.stream() - .map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).collect(Collectors.toList()), - readResultsProcessingExecutor); + Futures.transform(rs.allRows(readResultsProcessingExecutor), rows -> + rows.stream() + .map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).collect(Collectors.toList()), + readResultsProcessingExecutor); + } + + private ListenableFuture> getPartitionsFuture(TenantId tenantId, ReadTsKvQuery query, EntityId entityId, long minPartition, long maxPartition) { + if (isFixedPartitioning()) { //no need to fetch partitions from DB + return Futures.immediateFuture(FIXED_PARTITION); + } + TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); + return Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); } private AsyncFunction, List> getFetchChunksAsyncFunction(TenantId tenantId, EntityId entityId, String key, Aggregation aggregation, long startTs, long endTs) { @@ -319,49 +403,8 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem }; } - @Override - public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindLatestStmt().bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()); - stmtBuilder.setUuid(1, entityId.getId()); - stmtBuilder.setString(2, key); - BoundStatement stmt = stmtBuilder.build(); - log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); - return getFuture(executeAsyncRead(tenantId, stmt), rs -> convertResultToTsKvEntry(key, rs.one())); - } - - @Override - public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindAllLatestStmt().bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()); - stmtBuilder.setUuid(1, entityId.getId()); - BoundStatement stmt = stmtBuilder.build(); - log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); - return getFutureAsync(executeAsyncRead(tenantId, stmt), rs -> convertAsyncResultSetToTsKvEntryList(rs)); - } - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - List> futures = new ArrayList<>(); - ttl = computeTtl(ttl); - long partition = toPartitionTs(tsKvEntry.getTs()); - DataType type = tsKvEntry.getDataType(); - if (setNullValuesEnabled) { - processSetNullValues(tenantId, entityId, tsKvEntry, ttl, futures, partition, type); - } - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getSaveStmt(type) : getSaveTtlStmt(type)).bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()) - .setUuid(1, entityId.getId()) - .setString(2, tsKvEntry.getKey()) - .setLong(3, partition) - .setLong(4, tsKvEntry.getTs()); - addValue(tsKvEntry, stmtBuilder, 5); - if (ttl > 0) { - stmtBuilder.setInt(6, (int) ttl); - } - BoundStatement stmt = stmtBuilder.build(); - futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); - return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); + private boolean isFixedPartitioning() { + return tsFormat.getTruncateUnit().equals(ChronoUnit.FOREVER); } private void processSetNullValues(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl, List> futures, long partition, DataType type) { @@ -414,26 +457,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); } - @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - if (isFixedPartitioning()) { - return Futures.immediateFuture(null); - } - ttl = computeTtl(ttl); - long partition = toPartitionTs(tsKvEntryTs); - log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key); - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt()).bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()) - .setUuid(1, entityId.getId()) - .setLong(2, partition) - .setString(3, key); - if (ttl > 0) { - stmtBuilder.setInt(4, (int) ttl); - } - BoundStatement stmt = stmtBuilder.build(); - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); - } - private long computeTtl(long ttl) { if (systemTtl > 0) { if (ttl == 0) { @@ -445,53 +468,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return ttl; } - @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()) - .setUuid(1, entityId.getId()) - .setString(2, tsKvEntry.getKey()) - .setLong(3, tsKvEntry.getTs()) - .set(4, tsKvEntry.getBooleanValue().orElse(null), Boolean.class) - .set(5, tsKvEntry.getStrValue().orElse(null), String.class) - .set(6, tsKvEntry.getLongValue().orElse(null), Long.class) - .set(7, tsKvEntry.getDoubleValue().orElse(null), Double.class); - Optional jsonV = tsKvEntry.getJsonValue(); - if (jsonV.isPresent()) { - stmtBuilder.setString(8, tsKvEntry.getJsonValue().get()); - } else { - stmtBuilder.setToNull(8); - } - BoundStatement stmt = stmtBuilder.build(); - - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); - } - - @Override - public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long minPartition = toPartitionTs(query.getStartTs()); - long maxPartition = toPartitionTs(query.getEndTs()); - - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); - - Futures.addCallback(partitionsListFuture, new FutureCallback>() { - @Override - public void onSuccess(@Nullable List partitions) { - QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitions); - deleteAsync(tenantId, cursor, resultFuture); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); - } - }, readResultsProcessingExecutor); - return resultFuture; - } - private void deleteAsync(TenantId tenantId, final QueryCursor cursor, final SimpleListenableFuture resultFuture) { if (!cursor.hasNextPartition()) { resultFuture.set(null); @@ -534,119 +510,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return deleteStmt; } - @Override - public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture latestEntryFuture = findLatest(tenantId, entityId, query.getKey()); - - ListenableFuture booleanFuture = Futures.transform(latestEntryFuture, latestEntry -> { - long ts = latestEntry.getTs(); - if (ts > query.getStartTs() && ts <= query.getEndTs()) { - return true; - } else { - log.trace("Won't be deleted latest value for [{}], key - {}", entityId, query.getKey()); - } - return false; - }, readResultsProcessingExecutor); - - ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - return deleteLatest(tenantId, entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, readResultsProcessingExecutor); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - Futures.addCallback(removedLatestFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - if (query.getRewriteLatestIfDeleted()) { - ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - return getNewLatestEntryFuture(tenantId, entityId, query); - } - return Futures.immediateFuture(null); - }, readResultsProcessingExecutor); - - try { - resultFuture.set(savedLatestFuture.get()); - } catch (InterruptedException | ExecutionException e) { - log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); - } - } else { - resultFuture.set(null); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to process remove of the latest value", entityId, t); - } - }, MoreExecutors.directExecutor()); - return resultFuture; - } - - private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long startTs = 0; - long endTs = query.getStartTs() - 1; - ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, - Aggregation.NONE, DESC_ORDER); - ListenableFuture> future = findAllAsync(tenantId, entityId, findNewLatestQuery); - - return Futures.transformAsync(future, entryList -> { - if (entryList.size() == 1) { - return saveLatest(tenantId, entityId, entryList.get(0)); - } else { - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, readResultsProcessingExecutor); - } - - private ListenableFuture deleteLatest(TenantId tenantId, EntityId entityId, String key) { - Statement delete = QueryBuilder.deleteFrom(ModelConstants.TS_KV_LATEST_CF) - .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) - .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) - .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)).build(); - log.debug("Remove request: {}", delete.toString()); - return getFuture(executeAsyncWrite(tenantId, delete), rs -> null); - } - - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long minPartition = toPartitionTs(query.getStartTs()); - long maxPartition = toPartitionTs(query.getEndTs()); - if (minPartition == maxPartition) { - return Futures.immediateFuture(null); - } else { - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); - - Futures.addCallback(partitionsListFuture, new FutureCallback>() { - @Override - public void onSuccess(@Nullable List partitions) { - int index = 0; - if (minPartition != query.getStartTs()) { - index = 1; - } - List partitionsToDelete = new ArrayList<>(); - for (int i = index; i < partitions.size() - 1; i++) { - partitionsToDelete.add(partitions.get(i)); - } - QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitionsToDelete); - deletePartitionAsync(tenantId, cursor, resultFuture); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); - } - }, readResultsProcessingExecutor); - return resultFuture; - } - } - private void deletePartitionAsync(TenantId tenantId, final QueryCursor cursor, final SimpleListenableFuture resultFuture) { if (!cursor.hasNextPartition()) { resultFuture.set(null); @@ -685,79 +548,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return deletePartitionStmt; } - private ListenableFuture> convertAsyncResultSetToTsKvEntryList(TbResultSet rs) { - return Futures.transform(rs.allRows(readResultsProcessingExecutor), - rows -> this.convertResultToTsKvEntryList(rows), readResultsProcessingExecutor); - } - - private List convertResultToTsKvEntryList(List rows) { - List entries = new ArrayList<>(rows.size()); - if (!rows.isEmpty()) { - rows.forEach(row -> entries.add(convertResultToTsKvEntry(row))); - } - return entries; - } - - private TsKvEntry convertResultToTsKvEntry(String key, Row row) { - if (row != null) { - long ts = row.getLong(ModelConstants.TS_COLUMN); - return new BasicTsKvEntry(ts, toKvEntry(row, key)); - } else { - return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); - } - } - - private TsKvEntry convertResultToTsKvEntry(Row row) { - String key = row.getString(ModelConstants.KEY_COLUMN); - long ts = row.getLong(ModelConstants.TS_COLUMN); - return new BasicTsKvEntry(ts, toKvEntry(row, key)); - } - - public static KvEntry toKvEntry(Row row, String key) { - KvEntry kvEntry = null; - String strV = row.get(ModelConstants.STRING_VALUE_COLUMN, String.class); - if (strV != null) { - kvEntry = new StringDataEntry(key, strV); - } else { - Long longV = row.get(ModelConstants.LONG_VALUE_COLUMN, Long.class); - if (longV != null) { - kvEntry = new LongDataEntry(key, longV); - } else { - Double doubleV = row.get(ModelConstants.DOUBLE_VALUE_COLUMN, Double.class); - if (doubleV != null) { - kvEntry = new DoubleDataEntry(key, doubleV); - } else { - Boolean boolV = row.get(ModelConstants.BOOLEAN_VALUE_COLUMN, Boolean.class); - if (boolV != null) { - kvEntry = new BooleanDataEntry(key, boolV); - } else { - String jsonV = row.get(ModelConstants.JSON_VALUE_COLUMN, String.class); - if (StringUtils.isNoneEmpty(jsonV)) { - kvEntry = new JsonDataEntry(key, jsonV); - } else { - log.warn("All values in key-value row are nullable "); - } - } - } - } - } - return kvEntry; - } - - /** - * Select existing partitions from the table - * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity - */ - private TbResultSetFuture fetchPartitions(TenantId tenantId, EntityId entityId, String key, long minPartition, long maxPartition) { - Select select = QueryBuilder.selectFrom(ModelConstants.TS_KV_PARTITIONS_CF).column(ModelConstants.PARTITION_COLUMN) - .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) - .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) - .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)) - .whereColumn(ModelConstants.PARTITION_COLUMN).isGreaterThanOrEqualTo(literal(minPartition)) - .whereColumn(ModelConstants.PARTITION_COLUMN).isLessThanOrEqualTo(literal(maxPartition)); - return executeAsyncRead(tenantId, select.build()); - } - private PreparedStatement getSaveStmt(DataType dataType) { if (saveStmts == null) { saveStmts = new PreparedStatement[DataType.values().length]; @@ -792,63 +582,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return saveTtlStmts[dataType.ordinal()]; } - private PreparedStatement getFetchStmt(Aggregation aggType, String orderBy) { - switch (orderBy) { - case ASC_ORDER: - if (fetchStmtsAsc == null) { - fetchStmtsAsc = initFetchStmt(orderBy); - } - return fetchStmtsAsc[aggType.ordinal()]; - case DESC_ORDER: - if (fetchStmtsDesc == null) { - fetchStmtsDesc = initFetchStmt(orderBy); - } - return fetchStmtsDesc[aggType.ordinal()]; - default: - throw new RuntimeException("Not supported" + orderBy + "order!"); - } - } - - private PreparedStatement[] initFetchStmt(String orderBy) { - PreparedStatement[] fetchStmts = new PreparedStatement[Aggregation.values().length]; - for (Aggregation type : Aggregation.values()) { - if (type == Aggregation.SUM && fetchStmts[Aggregation.AVG.ordinal()] != null) { - fetchStmts[type.ordinal()] = fetchStmts[Aggregation.AVG.ordinal()]; - } else if (type == Aggregation.AVG && fetchStmts[Aggregation.SUM.ordinal()] != null) { - fetchStmts[type.ordinal()] = fetchStmts[Aggregation.SUM.ordinal()]; - } else { - fetchStmts[type.ordinal()] = prepare(SELECT_PREFIX + - String.join(", ", ModelConstants.getFetchColumnNames(type)) + " FROM " + ModelConstants.TS_KV_CF - + " WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.PARTITION_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.TS_COLUMN + " > ? " - + "AND " + ModelConstants.TS_COLUMN + " <= ?" - + (type == Aggregation.NONE ? " ORDER BY " + ModelConstants.TS_COLUMN + " " + orderBy + " LIMIT ?" : "")); - } - } - return fetchStmts; - } - - private PreparedStatement getLatestStmt() { - if (latestInsertStmt == null) { - latestInsertStmt = prepare(INSERT_INTO + ModelConstants.TS_KV_LATEST_CF + - "(" + ModelConstants.ENTITY_TYPE_COLUMN + - "," + ModelConstants.ENTITY_ID_COLUMN + - "," + ModelConstants.KEY_COLUMN + - "," + ModelConstants.TS_COLUMN + - "," + ModelConstants.BOOLEAN_VALUE_COLUMN + - "," + ModelConstants.STRING_VALUE_COLUMN + - "," + ModelConstants.LONG_VALUE_COLUMN + - "," + ModelConstants.DOUBLE_VALUE_COLUMN + - "," + ModelConstants.JSON_VALUE_COLUMN + ")" + - " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"); - } - return latestInsertStmt; - } - - private PreparedStatement getPartitionInsertStmt() { if (partitionInsertStmt == null) { partitionInsertStmt = prepare(INSERT_INTO + ModelConstants.TS_KV_PARTITIONS_CF + @@ -873,42 +606,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return partitionInsertTtlStmt; } - - private PreparedStatement getFindLatestStmt() { - if (findLatestStmt == null) { - findLatestStmt = prepare(SELECT_PREFIX + - ModelConstants.KEY_COLUMN + "," + - ModelConstants.TS_COLUMN + "," + - ModelConstants.STRING_VALUE_COLUMN + "," + - ModelConstants.BOOLEAN_VALUE_COLUMN + "," + - ModelConstants.LONG_VALUE_COLUMN + "," + - ModelConstants.DOUBLE_VALUE_COLUMN + "," + - ModelConstants.JSON_VALUE_COLUMN + " " + - "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + - "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + - "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + - "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM); - } - return findLatestStmt; - } - - private PreparedStatement getFindAllLatestStmt() { - if (findAllLatestStmt == null) { - findAllLatestStmt = prepare(SELECT_PREFIX + - ModelConstants.KEY_COLUMN + "," + - ModelConstants.TS_COLUMN + "," + - ModelConstants.STRING_VALUE_COLUMN + "," + - ModelConstants.BOOLEAN_VALUE_COLUMN + "," + - ModelConstants.LONG_VALUE_COLUMN + "," + - ModelConstants.DOUBLE_VALUE_COLUMN + "," + - ModelConstants.JSON_VALUE_COLUMN + " " + - "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + - "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + - "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM); - } - return findAllLatestStmt; - } - private static String getColumnName(DataType type) { switch (type) { case BOOLEAN: @@ -951,4 +648,56 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem } } + /** + // * Select existing partitions from the table + // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity + // */ + private TbResultSetFuture fetchPartitions(TenantId tenantId, EntityId entityId, String key, long minPartition, long maxPartition) { + Select select = QueryBuilder.selectFrom(ModelConstants.TS_KV_PARTITIONS_CF).column(ModelConstants.PARTITION_COLUMN) + .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) + .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) + .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)) + .whereColumn(ModelConstants.PARTITION_COLUMN).isGreaterThanOrEqualTo(literal(minPartition)) + .whereColumn(ModelConstants.PARTITION_COLUMN).isLessThanOrEqualTo(literal(maxPartition)); + return executeAsyncRead(tenantId, select.build()); + } + + private PreparedStatement getFetchStmt(Aggregation aggType, String orderBy) { + switch (orderBy) { + case ASC_ORDER: + if (fetchStmtsAsc == null) { + fetchStmtsAsc = initFetchStmt(orderBy); + } + return fetchStmtsAsc[aggType.ordinal()]; + case DESC_ORDER: + if (fetchStmtsDesc == null) { + fetchStmtsDesc = initFetchStmt(orderBy); + } + return fetchStmtsDesc[aggType.ordinal()]; + default: + throw new RuntimeException("Not supported" + orderBy + "order!"); + } + } + + private PreparedStatement[] initFetchStmt(String orderBy) { + PreparedStatement[] fetchStmts = new PreparedStatement[Aggregation.values().length]; + for (Aggregation type : Aggregation.values()) { + if (type == Aggregation.SUM && fetchStmts[Aggregation.AVG.ordinal()] != null) { + fetchStmts[type.ordinal()] = fetchStmts[Aggregation.AVG.ordinal()]; + } else if (type == Aggregation.AVG && fetchStmts[Aggregation.SUM.ordinal()] != null) { + fetchStmts[type.ordinal()] = fetchStmts[Aggregation.SUM.ordinal()]; + } else { + fetchStmts[type.ordinal()] = prepare(SELECT_PREFIX + + String.join(", ", ModelConstants.getFetchColumnNames(type)) + " FROM " + ModelConstants.TS_KV_CF + + " WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.PARTITION_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.TS_COLUMN + " > ? " + + "AND " + ModelConstants.TS_COLUMN + " <= ?" + + (type == Aggregation.NONE ? " ORDER BY " + ModelConstants.TS_COLUMN + " " + orderBy + " LIMIT ?" : "")); + } + } + return fetchStmts; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java new file mode 100644 index 0000000000..00eb44c7a2 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -0,0 +1,249 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.cql.Statement; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.nosql.TbResultSet; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; +import org.thingsboard.server.dao.util.NoSqlTsLatestDao; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; + +@Component +@Slf4j +@NoSqlTsLatestDao +public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimeseriesDao implements TimeseriesLatestDao { + + @Autowired + protected AggregationTimeseriesDao aggregationTimeseriesDao; + + private PreparedStatement latestInsertStmt; + private PreparedStatement findLatestStmt; + private PreparedStatement findAllLatestStmt; + + @Override + public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindLatestStmt().bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()); + stmtBuilder.setUuid(1, entityId.getId()); + stmtBuilder.setString(2, key); + BoundStatement stmt = stmtBuilder.build(); + log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); + return getFuture(executeAsyncRead(tenantId, stmt), rs -> convertResultToTsKvEntry(key, rs.one())); + } + + @Override + public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindAllLatestStmt().bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()); + stmtBuilder.setUuid(1, entityId.getId()); + BoundStatement stmt = stmtBuilder.build(); + log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); + return getFutureAsync(executeAsyncRead(tenantId, stmt), rs -> convertAsyncResultSetToTsKvEntryList(rs)); + } + + @Override + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()) + .setUuid(1, entityId.getId()) + .setString(2, tsKvEntry.getKey()) + .setLong(3, tsKvEntry.getTs()) + .set(4, tsKvEntry.getBooleanValue().orElse(null), Boolean.class) + .set(5, tsKvEntry.getStrValue().orElse(null), String.class) + .set(6, tsKvEntry.getLongValue().orElse(null), Long.class) + .set(7, tsKvEntry.getDoubleValue().orElse(null), Double.class); + Optional jsonV = tsKvEntry.getJsonValue(); + if (jsonV.isPresent()) { + stmtBuilder.setString(8, tsKvEntry.getJsonValue().get()); + } else { + stmtBuilder.setToNull(8); + } + BoundStatement stmt = stmtBuilder.build(); + + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); + } + + @Override + public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture latestEntryFuture = findLatest(tenantId, entityId, query.getKey()); + + ListenableFuture booleanFuture = Futures.transform(latestEntryFuture, latestEntry -> { + long ts = latestEntry.getTs(); + if (ts > query.getStartTs() && ts <= query.getEndTs()) { + return true; + } else { + log.trace("Won't be deleted latest value for [{}], key - {}", entityId, query.getKey()); + } + return false; + }, readResultsProcessingExecutor); + + ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + return deleteLatest(tenantId, entityId, query.getKey()); + } + return Futures.immediateFuture(null); + }, readResultsProcessingExecutor); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + Futures.addCallback(removedLatestFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + if (query.getRewriteLatestIfDeleted()) { + ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + return getNewLatestEntryFuture(tenantId, entityId, query); + } + return Futures.immediateFuture(null); + }, readResultsProcessingExecutor); + + try { + resultFuture.set(savedLatestFuture.get()); + } catch (InterruptedException | ExecutionException e) { + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); + } + } else { + resultFuture.set(null); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to process remove of the latest value", entityId, t); + } + }, MoreExecutors.directExecutor()); + return resultFuture; + } + + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long startTs = 0; + long endTs = query.getStartTs() - 1; + ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, + Aggregation.NONE, DESC_ORDER); + ListenableFuture> future = aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery); + + return Futures.transformAsync(future, entryList -> { + if (entryList.size() == 1) { + return saveLatest(tenantId, entityId, entryList.get(0)); + } else { + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); + } + return Futures.immediateFuture(null); + }, readResultsProcessingExecutor); + } + + private ListenableFuture deleteLatest(TenantId tenantId, EntityId entityId, String key) { + Statement delete = QueryBuilder.deleteFrom(ModelConstants.TS_KV_LATEST_CF) + .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) + .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) + .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)).build(); + log.debug("Remove request: {}", delete.toString()); + return getFuture(executeAsyncWrite(tenantId, delete), rs -> null); + } + + private ListenableFuture> convertAsyncResultSetToTsKvEntryList(TbResultSet rs) { + return Futures.transform(rs.allRows(readResultsProcessingExecutor), + rows -> this.convertResultToTsKvEntryList(rows), readResultsProcessingExecutor); + } + + private TsKvEntry convertResultToTsKvEntry(String key, Row row) { + if (row != null) { + long ts = row.getLong(ModelConstants.TS_COLUMN); + return new BasicTsKvEntry(ts, toKvEntry(row, key)); + } else { + return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); + } + } + + private PreparedStatement getLatestStmt() { + if (latestInsertStmt == null) { + latestInsertStmt = prepare(INSERT_INTO + ModelConstants.TS_KV_LATEST_CF + + "(" + ModelConstants.ENTITY_TYPE_COLUMN + + "," + ModelConstants.ENTITY_ID_COLUMN + + "," + ModelConstants.KEY_COLUMN + + "," + ModelConstants.TS_COLUMN + + "," + ModelConstants.BOOLEAN_VALUE_COLUMN + + "," + ModelConstants.STRING_VALUE_COLUMN + + "," + ModelConstants.LONG_VALUE_COLUMN + + "," + ModelConstants.DOUBLE_VALUE_COLUMN + + "," + ModelConstants.JSON_VALUE_COLUMN + ")" + + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"); + } + return latestInsertStmt; + } + + private PreparedStatement getFindLatestStmt() { + if (findLatestStmt == null) { + findLatestStmt = prepare(SELECT_PREFIX + + ModelConstants.KEY_COLUMN + "," + + ModelConstants.TS_COLUMN + "," + + ModelConstants.STRING_VALUE_COLUMN + "," + + ModelConstants.BOOLEAN_VALUE_COLUMN + "," + + ModelConstants.LONG_VALUE_COLUMN + "," + + ModelConstants.DOUBLE_VALUE_COLUMN + "," + + ModelConstants.JSON_VALUE_COLUMN + " " + + "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + + "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM); + } + return findLatestStmt; + } + + private PreparedStatement getFindAllLatestStmt() { + if (findAllLatestStmt == null) { + findAllLatestStmt = prepare(SELECT_PREFIX + + ModelConstants.KEY_COLUMN + "," + + ModelConstants.TS_COLUMN + "," + + ModelConstants.STRING_VALUE_COLUMN + "," + + ModelConstants.BOOLEAN_VALUE_COLUMN + "," + + ModelConstants.LONG_VALUE_COLUMN + "," + + ModelConstants.DOUBLE_VALUE_COLUMN + "," + + ModelConstants.JSON_VALUE_COLUMN + " " + + "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + + "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM); + } + return findAllLatestStmt; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index 39d965630e..dda2b227c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -31,19 +31,11 @@ public interface TimeseriesDao { ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); - ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key); - - ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); - ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); - ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); - ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java new file mode 100644 index 0000000000..7392e0e3d4 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface TimeseriesLatestDao { + + ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key); + + ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); + + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + + ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); + +} diff --git a/dao/src/main/resources/cassandra/schema-ts-latest.cql b/dao/src/main/resources/cassandra/schema-ts-latest.cql new file mode 100644 index 0000000000..623f530888 --- /dev/null +++ b/dao/src/main/resources/cassandra/schema-ts-latest.cql @@ -0,0 +1,34 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE KEYSPACE IF NOT EXISTS thingsboard +WITH replication = { + 'class' : 'SimpleStrategy', + 'replication_factor' : 1 +}; + +CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf ( + entity_type text, // (DEVICE, CUSTOMER, TENANT) + entity_id timeuuid, + key text, + ts bigint, + bool_v boolean, + str_v text, + long_v bigint, + dbl_v double, + json_v text, + PRIMARY KEY (( entity_type, entity_id ), key) +) WITH compaction = { 'class' : 'LeveledCompactionStrategy' }; diff --git a/dao/src/main/resources/cassandra/schema-ts.cql b/dao/src/main/resources/cassandra/schema-ts.cql index c0f4b74467..957c6ee6eb 100644 --- a/dao/src/main/resources/cassandra/schema-ts.cql +++ b/dao/src/main/resources/cassandra/schema-ts.cql @@ -42,16 +42,3 @@ CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf ( PRIMARY KEY (( entity_type, entity_id, key ), partition) ) WITH CLUSTERING ORDER BY ( partition ASC ) AND compaction = { 'class' : 'LeveledCompactionStrategy' }; - -CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf ( - entity_type text, // (DEVICE, CUSTOMER, TENANT) - entity_id timeuuid, - key text, - ts bigint, - bool_v boolean, - str_v text, - long_v bigint, - dbl_v double, - json_v text, - PRIMARY KEY (( entity_type, entity_id ), key) -) WITH compaction = { 'class' : 'LeveledCompactionStrategy' }; diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index f634590598..664925db76 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -272,3 +272,20 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); +CREATE TABLE IF NOT EXISTS ts_kv_latest ( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v varchar(10000000), + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) +); + +CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( + key varchar(255) NOT NULL, + key_id int GENERATED BY DEFAULT AS IDENTITY(start with 0 increment by 1) UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) +); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index af4d5d158c..50924bbbd4 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -14,6 +14,13 @@ -- limitations under the License. -- +CREATE TABLE IF NOT EXISTS tb_schema_settings +( + schema_version bigint NOT NULL, + CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) +); + +INSERT INTO tb_schema_settings (schema_version) VALUES (3001000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 3001000; CREATE TABLE IF NOT EXISTS admin_settings ( id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY, @@ -331,3 +338,4 @@ BEGIN '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); END; $$ LANGUAGE plpgsql; + diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index b8e62196a6..a9816f4a7a 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -46,14 +46,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest ( CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); -CREATE TABLE IF NOT EXISTS tb_schema_settings -( - schema_version bigint NOT NULL, - CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) -); - -INSERT INTO tb_schema_settings (schema_version) VALUES (2005001) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005001; - CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ BEGIN diff --git a/dao/src/main/resources/sql/schema-ts-hsql.sql b/dao/src/main/resources/sql/schema-ts-hsql.sql index ce4e4d4275..eee09516fc 100644 --- a/dao/src/main/resources/sql/schema-ts-hsql.sql +++ b/dao/src/main/resources/sql/schema-ts-hsql.sql @@ -28,18 +28,6 @@ CREATE TABLE IF NOT EXISTS ts_kv ( CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ); -CREATE TABLE IF NOT EXISTS ts_kv_latest ( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v varchar(10000000), - CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); - CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( key varchar(255) NOT NULL, key_id int GENERATED BY DEFAULT AS IDENTITY(start with 0 increment by 1) UNIQUE, diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 0b5cab6ebb..ef6c51aa0a 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -27,19 +27,6 @@ CREATE TABLE IF NOT EXISTS ts_kv CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ) PARTITION BY RANGE (ts); -CREATE TABLE IF NOT EXISTS ts_kv_latest -( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, - CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); - CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( key varchar(255) NOT NULL, @@ -47,14 +34,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_dictionary CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) ); -CREATE TABLE IF NOT EXISTS tb_schema_settings -( - schema_version bigint NOT NULL, - CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) -); - -INSERT INTO tb_schema_settings (schema_version) VALUES (2005001) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005001; - CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ diff --git a/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java b/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java index 3dae53bb83..f7d45ff13d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java +++ b/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java @@ -22,7 +22,6 @@ import org.dbunit.ext.hsqldb.HsqldbDataTypeFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.thingsboard.server.dao.util.SqlDao; import javax.sql.DataSource; import java.io.IOException; @@ -32,7 +31,6 @@ import java.sql.SQLException; * Created by Valerii Sosliuk on 5/6/2017. */ @Configuration -@SqlDao public class JpaDbunitTestConfig { @Autowired diff --git a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java index 824f5887e2..1f467209e5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java @@ -40,7 +40,8 @@ public class NoSqlDaoServiceTestSuite { public static CustomCassandraCQLUnit cassandraUnit = new CustomCassandraCQLUnit( Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false) + new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), + new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) ), "cassandra-test.yaml", 30000L); diff --git a/dao/src/test/resources/nosql-test.properties b/dao/src/test/resources/nosql-test.properties index 8f827eab43..eefa4f0b95 100644 --- a/dao/src/test/resources/nosql-test.properties +++ b/dao/src/test/resources/nosql-test.properties @@ -1,4 +1,5 @@ database.ts.type=cassandra +database.ts_latest.type=cassandra sql.ts_inserts_executor_type=fixed sql.ts_inserts_fixed_thread_pool_size=10 diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 11ca8b8daf..123c0506cc 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -1,4 +1,5 @@ database.ts.type=sql +database.ts_latest.type=sql sql.ts_inserts_executor_type=fixed sql.ts_inserts_fixed_thread_pool_size=200