From 84cb471e0d7e2e49807a232b87c949d44e377fb2 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 29 Jan 2020 12:53:46 +0200 Subject: [PATCH] Sql timeseries improvements (#2033) * init commit * cleaned code and add test-properties * cleaned code * psql-update * timescale-update * code-refactoring * fix typo * renamed dao * revert indents * refactored code * fix typo * init-partitioning * code updated * cleaned code * fixed license * fix typo * fixed code after review * add annotation to repository * update psql version for docker * postgres-10 * postgres-10 * update docker compose config * fixed partition saving * change key_id to serial column definition * upgrade psql added * add separate upgrade service * added upgrade script * change image on k8s * change logs * resolve conflict after merge with master * revert datasource url in yml * fix typo * license header fix * remove old methods for the timeseries inserts * clean up code * fix saveOrUpdate for PostgreSQL * refactoring & revert Timescale to use latest table * added PsqlTsAnyDao * duplicated code method removed * remove unused invert dictionary map * change the upgrade directory from 2.4.1 to 2.4.3 * refactor JpaPsqlTimeseriesDao --- .../upgrade/2.4.3/schema_update_psql_ts.sql | 179 +++++++ .../install/ThingsboardInstallService.java | 35 +- .../CassandraDatabaseUpgradeService.java | 2 +- .../DatabaseEntitiesUpgradeService.java | 22 + ...ice.java => DatabaseTsUpgradeService.java} | 4 +- ....java => HsqlTsDatabaseSchemaService.java} | 8 +- .../install/PsqlTsDatabaseSchemaService.java | 32 ++ .../install/PsqlTsDatabaseUpgradeService.java | 145 ++++++ .../install/SqlDatabaseUpgradeService.java | 8 +- .../src/main/resources/thingsboard.yml | 4 +- .../controller/ControllerSqlTestSuite.java | 2 +- .../server/mqtt/MqttSqlTestSuite.java | 2 +- .../server/rules/RuleEngineSqlTestSuite.java | 2 +- .../server/system/SystemSqlTestSuite.java | 2 +- .../server/dao/util/PsqlTsAnyDao.java | 23 + ...lTsDaoConfig.java => HsqlTsDaoConfig.java} | 10 +- .../server/dao/PsqlTsDaoConfig.java | 37 ++ .../server/dao/TimescaleDaoConfig.java | 6 +- .../dao/audit/CassandraAuditLogDao.java | 6 +- .../server/dao/model/ModelConstants.java | 1 + .../dao/model/sql/AbstractTsKvEntity.java | 30 +- .../sqlts/dictionary/TsKvDictionary.java | 45 ++ .../TsKvDictionaryCompositeKey.java | 34 ++ .../sqlts/{ts => hsql}/TsKvCompositeKey.java | 3 +- .../model/sqlts/{ts => hsql}/TsKvEntity.java | 32 +- .../TsKvLatestCompositeKey.java | 2 +- .../{ts => latest}/TsKvLatestEntity.java | 38 +- .../model/sqlts/psql/TsKvCompositeKey.java | 37 ++ .../dao/model/sqlts/psql/TsKvEntity.java | 135 ++++++ .../timescale/TimescaleTsKvCompositeKey.java | 7 +- .../sqlts/timescale/TimescaleTsKvEntity.java | 46 +- ...paAbstractDaoListeningExecutorService.java | 5 - .../dao/sqlts/AbstractInsertRepository.java | 51 -- .../sqlts/AbstractLatestInsertRepository.java | 58 --- .../sqlts/AbstractSimpleSqlTimeseriesDao.java | 156 +++++++ .../dao/sqlts/AbstractSqlTimeseriesDao.java | 163 ++++++- .../AbstractTimeseriesInsertRepository.java | 58 --- .../server/dao/sqlts/EntityContainer.java | 29 ++ .../dao/sqlts/InsertLatestRepository.java | 26 ++ .../server/dao/sqlts/InsertTsRepository.java | 26 ++ .../dictionary/TsKvDictionaryRepository.java | 30 ++ .../hsql/HsqlTimeseriesInsertRepository.java | 89 ++++ .../dao/sqlts/hsql/JpaHsqlTimeseriesDao.java | 206 +++++++++ .../TsKvHsqlRepository.java} | 10 +- .../latest/HsqlLatestInsertRepository.java | 85 ++++ .../PsqlLatestInsertRepository.java | 72 +-- .../{ts => latest}/TsKvLatestRepository.java | 6 +- .../dao/sqlts/psql/JpaPsqlTimeseriesDao.java | 312 +++++++++++++ .../psql/PsqlPartitioningRepository.java | 41 ++ .../psql/PsqlTimeseriesInsertRepository.java | 101 ++++ .../dao/sqlts/psql/TsKvPsqlRepository.java | 132 ++++++ .../timescale/AggregationRepository.java | 15 +- .../timescale/TimescaleInsertRepository.java | 78 +--- .../timescale/TimescaleTimeseriesDao.java | 231 +++++----- .../timescale/TsKvTimescaleRepository.java | 23 +- .../sqlts/ts/HsqlLatestInsertRepository.java | 140 ------ .../ts/HsqlTimeseriesInsertRepository.java | 141 ------ .../server/dao/sqlts/ts/JpaTimeseriesDao.java | 436 ------------------ .../ts/PsqlTimeseriesInsertRepository.java | 143 ------ .../CassandraBaseTimeseriesDao.java | 4 +- ...ionDate.java => NoSqlTsPartitionDate.java} | 10 +- .../server/dao/timeseries/PsqlPartition.java | 43 ++ .../dao/timeseries/SqlTsPartitionDate.java | 93 ++++ .../main/resources/sql/schema-timescale.sql | 22 +- .../sql/{schema-ts.sql => schema-ts-hsql.sql} | 0 dao/src/main/resources/sql/schema-ts-psql.sql | 43 ++ .../server/dao/AbstractJpaDaoTest.java | 2 +- .../server/dao/JpaDaoTestSuite.java | 16 +- .../server/dao/SqlDaoServiceTestSuite.java | 16 +- dao/src/test/resources/sql-test.properties | 24 +- .../sql/timescale/drop-all-tables.sql | 1 + docker/docker-compose.postgres.yml | 2 +- k8s/postgres.yml | 2 +- msa/tb/docker-postgres/start-db.sh | 4 +- msa/tb/docker-postgres/stop-db.sh | 2 +- 75 files changed, 2669 insertions(+), 1417 deletions(-) create mode 100644 application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql create mode 100644 application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java rename application/src/main/java/org/thingsboard/server/service/install/{DatabaseUpgradeService.java => DatabaseTsUpgradeService.java} (94%) rename application/src/main/java/org/thingsboard/server/service/install/{SqlTsDatabaseSchemaService.java => HsqlTsDatabaseSchemaService.java} (80%) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java rename dao/src/main/java/org/thingsboard/server/dao/{SqlTsDaoConfig.java => HsqlTsDaoConfig.java} (74%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/{ts => hsql}/TsKvCompositeKey.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/{ts => hsql}/TsKvEntity.java (78%) rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/{ts => latest}/TsKvLatestCompositeKey.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/{ts => latest}/TsKvLatestEntity.java (60%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sqlts/psql/TsKvCompositeKey.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sqlts/psql/TsKvEntity.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractLatestInsertRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSimpleSqlTimeseriesDao.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractTimeseriesInsertRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/EntityContainer.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertLatestRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertTsRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/HsqlTimeseriesInsertRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java rename dao/src/main/java/org/thingsboard/server/dao/sqlts/{ts/TsKvRepository.java => hsql/TsKvHsqlRepository.java} (96%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/HsqlLatestInsertRepository.java rename dao/src/main/java/org/thingsboard/server/dao/sqlts/{ts => latest}/PsqlLatestInsertRepository.java (64%) rename dao/src/main/java/org/thingsboard/server/dao/sqlts/{ts => latest}/TsKvLatestRepository.java (83%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlPartitioningRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlTimeseriesInsertRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/TsKvPsqlRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlLatestInsertRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlTimeseriesInsertRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/JpaTimeseriesDao.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java rename dao/src/main/java/org/thingsboard/server/dao/timeseries/{TsPartitionDate.java => NoSqlTsPartitionDate.java} (87%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java rename dao/src/main/resources/sql/{schema-ts.sql => schema-ts-hsql.sql} (100%) create mode 100644 dao/src/main/resources/sql/schema-ts-psql.sql diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql new file mode 100644 index 0000000000..07e0b51511 --- /dev/null +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql @@ -0,0 +1,179 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- load function check_version() + +CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$ +DECLARE + current_version integer; + valid_version boolean; +BEGIN + RAISE NOTICE 'Check the current installed PostgreSQL version...'; + SELECT current_setting('server_version_num') INTO current_version; + IF current_version < 100000 THEN + valid_version := FALSE; + ELSE + valid_version := TRUE; + END IF; + IF valid_version = FALSE THEN + RAISE NOTICE 'Postgres version should be at least more than 10!'; + ELSE + RAISE NOTICE 'PostgreSQL version is valid!'; + RAISE NOTICE 'Schema update started...'; + END IF; + RETURN valid_version; +END; +$$ LANGUAGE 'plpgsql'; + +-- load function create_partition_table() + +CREATE OR REPLACE FUNCTION create_partition_table() RETURNS VOID AS $$ + +BEGIN + ALTER TABLE ts_kv + RENAME TO ts_kv_old; + CREATE TABLE IF NOT EXISTS ts_kv + ( + LIKE ts_kv_old + ) + PARTITION BY RANGE (ts); + ALTER TABLE ts_kv + DROP COLUMN entity_type; + ALTER TABLE ts_kv + ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; + ALTER TABLE ts_kv + ALTER COLUMN key TYPE integer USING key::integer; +END; +$$ LANGUAGE 'plpgsql'; + + +-- load function create_partitions() + +CREATE OR REPLACE FUNCTION create_partitions() RETURNS VOID AS +$$ +DECLARE + partition_date varchar; + from_ts bigint; + to_ts bigint; + key_cursor CURSOR FOR select SUBSTRING(month_date.first_date, 1, 7) AS partition_date, + extract(epoch from (month_date.first_date)::timestamp) * 1000 as from_ts, + extract(epoch from (month_date.first_date::date + INTERVAL '1 MONTH')::timestamp) * + 1000 as to_ts + FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_01') AS first_date + FROM ts_kv_old) AS month_date; +BEGIN + OPEN key_cursor; + LOOP + FETCH key_cursor INTO partition_date, from_ts, to_ts; + EXIT WHEN NOT FOUND; + EXECUTE 'CREATE TABLE IF NOT EXISTS ts_kv_' || partition_date || + ' PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (' || from_ts || + ') TO (' || to_ts || ');'; + RAISE NOTICE 'A partition % has been created!',CONCAT('ts_kv_', partition_date); + END LOOP; + + CLOSE key_cursor; +END; +$$ language 'plpgsql'; + +-- load function create_ts_kv_dictionary_table() + +CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$ + +BEGIN + CREATE TABLE IF NOT EXISTS ts_kv_dictionary + ( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) + ); +END; +$$ LANGUAGE 'plpgsql'; + +-- load function insert_into_dictionary() + +CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS +$$ +DECLARE + insert_record RECORD; + key_cursor CURSOR FOR SELECT DISTINCT key + FROM ts_kv_old + ORDER BY key; +BEGIN + OPEN key_cursor; + LOOP + FETCH key_cursor INTO insert_record; + EXIT WHEN NOT FOUND; + IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN + INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key); + RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key; + ELSE + RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key; + END IF; + END LOOP; + CLOSE key_cursor; +END; +$$ language 'plpgsql'; + +-- load function insert_into_ts_kv() + +CREATE OR REPLACE FUNCTION insert_into_ts_kv() RETURNS void AS +$$ +DECLARE + insert_size CONSTANT integer := 10000; + insert_counter integer DEFAULT 0; + insert_record RECORD; + insert_cursor CURSOR FOR SELECT CONCAT(first, '-', second, '-1', third, '-', fourth, '-', fifth)::uuid AS entity_id, + substrings.key AS key, + substrings.ts AS ts, + substrings.bool_v AS bool_v, + substrings.str_v AS str_v, + substrings.long_v AS long_v, + substrings.dbl_v AS dbl_v + FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first, + SUBSTRING(entity_id, 4, 4) AS second, + SUBSTRING(entity_id, 1, 3) AS third, + SUBSTRING(entity_id, 16, 4) AS fourth, + SUBSTRING(entity_id, 20) AS fifth, + key_id AS key, + ts, + bool_v, + str_v, + long_v, + dbl_v + FROM ts_kv_old + INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS substrings; +BEGIN + OPEN insert_cursor; + LOOP + insert_counter := insert_counter + 1; + FETCH insert_cursor INTO insert_record; + IF NOT FOUND THEN + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1; + EXIT; + END IF; + INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) + VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, + insert_record.long_v, insert_record.dbl_v); + IF MOD(insert_counter, insert_size) = 0 THEN + RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter; + END IF; + END LOOP; + CLOSE insert_cursor; +END; +$$ LANGUAGE 'plpgsql'; + + diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index f424e262eb..78906e855e 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -23,7 +23,8 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; -import org.thingsboard.server.service.install.DatabaseUpgradeService; +import org.thingsboard.server.service.install.DatabaseTsUpgradeService; +import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; @@ -50,7 +51,10 @@ public class ThingsboardInstallService { private TsDatabaseSchemaService tsDatabaseSchemaService; @Autowired - private DatabaseUpgradeService databaseUpgradeService; + private DatabaseEntitiesUpgradeService databaseEntitiesUpgradeService; + + @Autowired + private DatabaseTsUpgradeService databaseTsUpgradeService; @Autowired private ComponentDiscoveryService componentDiscoveryService; @@ -73,48 +77,48 @@ public class ThingsboardInstallService { case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion log.info("Upgrading ThingsBoard from version 1.2.3 to 1.3.0 ..."); - databaseUpgradeService.upgradeDatabase("1.2.3"); + databaseEntitiesUpgradeService.upgradeDatabase("1.2.3"); case "1.3.0": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion log.info("Upgrading ThingsBoard from version 1.3.0 to 1.3.1 ..."); - databaseUpgradeService.upgradeDatabase("1.3.0"); + databaseEntitiesUpgradeService.upgradeDatabase("1.3.0"); case "1.3.1": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion log.info("Upgrading ThingsBoard from version 1.3.1 to 1.4.0 ..."); - databaseUpgradeService.upgradeDatabase("1.3.1"); + databaseEntitiesUpgradeService.upgradeDatabase("1.3.1"); case "1.4.0": log.info("Upgrading ThingsBoard from version 1.4.0 to 2.0.0 ..."); - databaseUpgradeService.upgradeDatabase("1.4.0"); + databaseEntitiesUpgradeService.upgradeDatabase("1.4.0"); dataUpdateService.updateData("1.4.0"); case "2.0.0": log.info("Upgrading ThingsBoard from version 2.0.0 to 2.1.1 ..."); - databaseUpgradeService.upgradeDatabase("2.0.0"); + databaseEntitiesUpgradeService.upgradeDatabase("2.0.0"); case "2.1.1": log.info("Upgrading ThingsBoard from version 2.1.1 to 2.1.2 ..."); - databaseUpgradeService.upgradeDatabase("2.1.1"); + databaseEntitiesUpgradeService.upgradeDatabase("2.1.1"); case "2.1.3": log.info("Upgrading ThingsBoard from version 2.1.3 to 2.2.0 ..."); - databaseUpgradeService.upgradeDatabase("2.1.3"); + databaseEntitiesUpgradeService.upgradeDatabase("2.1.3"); case "2.3.0": log.info("Upgrading ThingsBoard from version 2.3.0 to 2.3.1 ..."); - databaseUpgradeService.upgradeDatabase("2.3.0"); + databaseEntitiesUpgradeService.upgradeDatabase("2.3.0"); case "2.3.1": log.info("Upgrading ThingsBoard from version 2.3.1 to 2.4.0 ..."); - databaseUpgradeService.upgradeDatabase("2.3.1"); + databaseEntitiesUpgradeService.upgradeDatabase("2.3.1"); case "2.4.0": log.info("Upgrading ThingsBoard from version 2.4.0 to 2.4.1 ..."); @@ -122,11 +126,16 @@ public class ThingsboardInstallService { case "2.4.1": log.info("Upgrading ThingsBoard from version 2.4.1 to 2.4.2 ..."); - databaseUpgradeService.upgradeDatabase("2.4.1"); + databaseEntitiesUpgradeService.upgradeDatabase("2.4.1"); case "2.4.2": log.info("Upgrading ThingsBoard from version 2.4.2 to 2.4.3 ..."); - databaseUpgradeService.upgradeDatabase("2.4.2"); + databaseEntitiesUpgradeService.upgradeDatabase("2.4.2"); + + case "2.4.3": + log.info("Upgrading ThingsBoard from version 2.4.3 to 2.5 ..."); + + databaseTsUpgradeService.upgradeDatabase("2.4.3"); log.info("Updating system data..."); diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java index 285a191c04..7e05179be0 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java @@ -59,7 +59,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; @NoSqlDao @Profile("install") @Slf4j -public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService { +public class CassandraDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { private static final String SCHEMA_UPDATE_CQL = "schema_update.cql"; diff --git a/application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java new file mode 100644 index 0000000000..7abb97a6a9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +public interface DatabaseEntitiesUpgradeService { + + void upgradeDatabase(String fromVersion) throws Exception; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java rename to application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java index 47a3944e74..fe82cabff9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.install; -public interface DatabaseUpgradeService { +public interface DatabaseTsUpgradeService { void upgradeDatabase(String fromVersion) throws Exception; -} +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java rename to application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java index 1288b0d652..3c877ca636 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/HsqlTsDatabaseSchemaService.java @@ -17,14 +17,16 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.HsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; @Service @SqlTsDao +@HsqlDao @Profile("install") -public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService +public class HsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { - public SqlTsDatabaseSchemaService() { - super("schema-ts.sql", null); + public HsqlTsDatabaseSchemaService() { + super("schema-ts-hsql.sql", null); } } \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java new file mode 100644 index 0000000000..15b1e45247 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +@Service +@SqlTsDao +@PsqlDao +@Profile("install") +public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService + implements TsDatabaseSchemaService { + public PsqlTsDatabaseSchemaService() { + super("schema-ts-psql.sql", null); + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java new file mode 100644 index 0000000000..10b1e45231 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -0,0 +1,145 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Types; + +@Service +@Profile("install") +@Slf4j +@SqlTsDao +@PsqlDao +public class PsqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { + + private static final String CALL_REGEX = "call "; + private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; + private static final String CHECK_VERSION = CALL_REGEX + "check_version()"; + private static final String CREATE_PARTITION_TABLE = CALL_REGEX + "create_partition_table()"; + private static final String CREATE_PARTITIONS = CALL_REGEX + "create_partitions()"; + private static final String CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + "create_ts_kv_dictionary_table()"; + private static final String INSERT_INTO_DICTIONARY = CALL_REGEX + "insert_into_dictionary()"; + private static final String INSERT_INTO_TS_KV = CALL_REGEX + "insert_into_ts_kv()"; + private static final String DROP_OLD_TABLE = "DROP TABLE ts_kv_old;"; + + @Value("${spring.datasource.url}") + private String dbUrl; + + @Value("${spring.datasource.username}") + private String dbUserName; + + @Value("${spring.datasource.password}") + private String dbPassword; + + @Autowired + private InstallScripts installScripts; + + @Override + public void upgradeDatabase(String fromVersion) throws Exception { + switch (fromVersion) { + case "2.4.3": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating timeseries schema ..."); + log.info("Load upgrade functions ..."); + loadSql(conn); + log.info("Upgrade functions successfully loaded!"); + boolean versionValid = checkVersion(conn); + if (!versionValid) { + log.info("PostgreSQL version should be at least more than 10!"); + log.info("Please upgrade your PostgreSQL and restart the script!"); + } else { + log.info("PostgreSQL version is valid!"); + log.info("Updating schema ..."); + executeFunction(conn, CREATE_PARTITION_TABLE); + executeFunction(conn, CREATE_PARTITIONS); + executeFunction(conn, CREATE_TS_KV_DICTIONARY_TABLE); + executeFunction(conn, INSERT_INTO_DICTIONARY); + executeFunction(conn, INSERT_INTO_TS_KV); + dropOldTable(conn, DROP_OLD_TABLE); + log.info("schema timeseries updated!"); + } + } + break; + default: + throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); + } + } + + private void loadSql(Connection conn) { + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL); + try { + loadFunctions(schemaUpdateFile, conn); + } catch (Exception e) { + log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); + } + } + + private void loadFunctions(Path sqlFile, Connection conn) throws Exception { + String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); + conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + } + + private boolean checkVersion(Connection conn) { + log.info("Check the current PostgreSQL version..."); + boolean versionValid = false; + try { + CallableStatement callableStatement = conn.prepareCall("{? = " + CHECK_VERSION + " }"); + callableStatement.registerOutParameter(1, Types.BOOLEAN); + callableStatement.execute(); + versionValid = callableStatement.getBoolean(1); + callableStatement.close(); + } catch (Exception e) { + log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage()); + } + return versionValid; + } + + private void executeFunction(Connection conn, String query) { + log.info("{} ... ", query); + try { + CallableStatement callableStatement = conn.prepareCall("{" + query + "}"); + callableStatement.execute(); + callableStatement.close(); + log.info("Successfully executed: {}", query.replace(CALL_REGEX, "")); + } catch (Exception e) { + log.info("Failed to execute {} due to: {}", query, e.getMessage()); + } + } + + private void dropOldTable(Connection conn, String query) { + try { + conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + Thread.sleep(5000); + } catch (InterruptedException | SQLException e) { + log.info("Failed to drop table {} due to: {}", query.replace("DROP TABLE ", ""), e.getMessage()); + } + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index b855b09354..d018c7fcef 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -54,7 +54,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; @Profile("install") @Slf4j @SqlDao -public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { +public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { private static final String SCHEMA_UPDATE_SQL = "schema_update.sql"; @@ -172,7 +172,8 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { loadSql(schemaUpdateFile, conn); try { conn.createStatement().execute("ALTER TABLE device ADD COLUMN label varchar(255)"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - } catch (Exception e) {} + } catch (Exception e) { + } log.info("Schema updated."); } break; @@ -201,7 +202,8 @@ public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { log.info("Updating schema ..."); try { conn.createStatement().execute("ALTER TABLE alarm ADD COLUMN propagate_relation_types varchar"); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - } catch (Exception e) {} + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f374e1871e..8b934cd48f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -171,7 +171,7 @@ cassandra: read_consistency_level: "${CASSANDRA_READ_CONSISTENCY_LEVEL:ONE}" write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}" default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}" - # Specify partitioning size for timestamp key-value storage. Example MINUTES, HOURS, DAYS, MONTHS,INDEFINITE + # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS,INDEFINITE ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" ts_key_value_ttl: "${TS_KV_TTL:0}" events_ttl: "${TS_EVENTS_TTL:0}" @@ -214,6 +214,8 @@ sql: stats_print_interval_ms: "${SQL_TS_TIMESCALE_BATCH_STATS_PRINT_MS:10000}" # Specify whether to remove null characters from strValue of attributes and timeseries before insert remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" + # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE + ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" # Actor system parameters actors: diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java index af839fea94..4fe33e4716 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java @@ -30,7 +30,7 @@ public class ControllerSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java index 9f3c2406f3..5fb8c4d0c7 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java @@ -29,7 +29,7 @@ public class MqttSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); } diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java index f30cd661c4..ce2c6852be 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java @@ -30,7 +30,7 @@ public class RuleEngineSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); } diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java index 49777cea15..3cbb7d9773 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java @@ -31,7 +31,7 @@ public class SystemSqlTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java new file mode 100644 index 0000000000..b795ce451c --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +@ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.entities.type}'=='timescale') " + + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") +public @interface PsqlTsAnyDao { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java similarity index 74% rename from dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java rename to dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java index dea9c73f75..833c3745b9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java @@ -21,15 +21,17 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.HsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; @Configuration @EnableAutoConfiguration -@ComponentScan("org.thingsboard.server.dao.sqlts.ts") -@EnableJpaRepositories("org.thingsboard.server.dao.sqlts.ts") -@EntityScan("org.thingsboard.server.dao.model.sqlts.ts") +@ComponentScan({"org.thingsboard.server.dao.sqlts.hsql", "org.thingsboard.server.dao.sqlts.latest"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.hsql", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.hsql", "org.thingsboard.server.dao.model.sqlts.latest"}) @EnableTransactionManagement @SqlTsDao -public class SqlTsDaoConfig { +@HsqlDao +public class HsqlTsDaoConfig { } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java new file mode 100644 index 0000000000..e3caf5e3d3 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +@Configuration +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.psql", "org.thingsboard.server.dao.sqlts.latest"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.psql", "org.thingsboard.server.dao.sqlts.latest", "org.thingsboard.server.dao.sqlts.dictionary"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.psql", "org.thingsboard.server.dao.model.sqlts.latest", "org.thingsboard.server.dao.model.sqlts.dictionary"}) +@EnableTransactionManagement +@SqlTsDao +@PsqlDao +public class PsqlTsDaoConfig { + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java index bcb6ae7107..f2aa68c8db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java @@ -25,9 +25,9 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; @Configuration @EnableAutoConfiguration -@ComponentScan("org.thingsboard.server.dao.sqlts.timescale") -@EnableJpaRepositories("org.thingsboard.server.dao.sqlts.timescale") -@EntityScan("org.thingsboard.server.dao.model.sqlts.timescale") +@ComponentScan({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.latest"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.dictionary", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale", "org.thingsboard.server.dao.model.sqlts.dictionary", "org.thingsboard.server.dao.model.sqlts.latest"}) @EnableTransactionManagement @TimescaleDBTsDao public class TimescaleDaoConfig { diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/CassandraAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/audit/CassandraAuditLogDao.java index b92b7613dd..a2dc184936 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/CassandraAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/CassandraAuditLogDao.java @@ -41,7 +41,7 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.nosql.AuditLogEntity; import org.thingsboard.server.dao.nosql.CassandraAbstractSearchTimeDao; -import org.thingsboard.server.dao.timeseries.TsPartitionDate; +import org.thingsboard.server.dao.timeseries.NoSqlTsPartitionDate; import org.thingsboard.server.dao.util.NoSqlDao; import javax.annotation.Nullable; @@ -92,7 +92,7 @@ public class CassandraAuditLogDao extends CassandraAbstractSearchTimeDao partition = TsPartitionDate.parse(partitioning); + Optional partition = NoSqlTsPartitionDate.parse(partitioning); if (partition.isPresent()) { tsFormat = partition.get(); } else { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index b69e7c29d7..a07c4868e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -359,6 +359,7 @@ public class ModelConstants { public static final String PARTITION_COLUMN = "partition"; public static final String KEY_COLUMN = "key"; + public static final String KEY_ID_COLUMN = "key_id"; public static final String TS_COLUMN = "ts"; /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java index 1f4469edd2..d1a8f9c462 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java @@ -16,11 +16,6 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; import javax.persistence.Column; import javax.persistence.Id; @@ -28,10 +23,9 @@ import javax.persistence.MappedSuperclass; import static org.thingsboard.server.dao.model.ModelConstants.BOOLEAN_VALUE_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.DOUBLE_VALUE_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.LONG_VALUE_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data @MappedSuperclass @@ -43,12 +37,8 @@ public abstract class AbstractTsKvEntity { protected static final String MAX = "MAX"; @Id - @Column(name = ENTITY_ID_COLUMN) - protected String entityId; - - @Id - @Column(name = KEY_COLUMN) - protected String key; + @Column(name = TS_COLUMN) + protected Long ts; @Column(name = BOOLEAN_VALUE_COLUMN) protected Boolean booleanValue; @@ -62,20 +52,6 @@ public abstract class AbstractTsKvEntity { @Column(name = DOUBLE_VALUE_COLUMN) protected Double doubleValue; - protected KvEntry getKvEntry() { - KvEntry kvEntry = null; - if (strValue != null) { - kvEntry = new StringDataEntry(key, strValue); - } else if (longValue != null) { - kvEntry = new LongDataEntry(key, longValue); - } else if (doubleValue != null) { - kvEntry = new DoubleDataEntry(key, doubleValue); - } else if (booleanValue != null) { - kvEntry = new BooleanDataEntry(key, booleanValue); - } - return kvEntry; - } - public abstract boolean isNotEmpty(); protected static boolean isAllNull(Object... args) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java new file mode 100644 index 0000000000..c324051a8d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sqlts.dictionary; + +import lombok.Data; +import org.hibernate.annotations.Generated; +import org.hibernate.annotations.GenerationTime; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.IdClass; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; + +@Data +@Entity +@Table(name = "ts_kv_dictionary") +@IdClass(TsKvDictionaryCompositeKey.class) +public final class TsKvDictionary { + + @Id + @Column(name = KEY_COLUMN) + private String key; + + @Column(name = KEY_ID_COLUMN, unique = true, columnDefinition="serial") + @Generated(GenerationTime.INSERT) + private int keyId; + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java new file mode 100644 index 0000000000..064f5ce46c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sqlts.dictionary; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.persistence.Transient; +import java.io.Serializable; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TsKvDictionaryCompositeKey implements Serializable{ + + @Transient + private static final long serialVersionUID = -4089175869616037523L; + + private String key; +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvCompositeKey.java similarity index 95% rename from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvCompositeKey.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvCompositeKey.java index 7eba9a9041..0d1b7f57a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvCompositeKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvCompositeKey.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.model.sqlts.ts; +package org.thingsboard.server.dao.model.sqlts.hsql; import lombok.AllArgsConstructor; import lombok.Data; @@ -35,4 +35,5 @@ public class TsKvCompositeKey implements Serializable { private String entityId; private String key; private long ts; + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvEntity.java similarity index 78% rename from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvEntity.java index ba12b5d5cd..97e68655ad 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/hsql/TsKvEntity.java @@ -13,11 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.model.sqlts.ts; +package org.thingsboard.server.dao.model.sqlts.hsql; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ToData; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; @@ -30,8 +35,9 @@ import javax.persistence.Id; import javax.persistence.IdClass; import javax.persistence.Table; +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; @Data @Entity @@ -45,8 +51,12 @@ public final class TsKvEntity extends AbstractTsKvEntity implements ToData { + + @Id + @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid") + protected UUID entityId; + + @Id + @Column(name = KEY_COLUMN) + protected int key; + + @Transient + protected String strKey; + + public TsKvEntity() { + } + + public TsKvEntity(String strValue) { + this.strValue = strValue; + } + + public TsKvEntity(Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String aggType) { + if (!isAllNull(longValue, doubleValue, longCountValue, doubleCountValue)) { + switch (aggType) { + case AVG: + double sum = 0.0; + if (longValue != null) { + sum += longValue; + } + if (doubleValue != null) { + sum += doubleValue; + } + long totalCount = longCountValue + doubleCountValue; + if (totalCount > 0) { + this.doubleValue = sum / (longCountValue + doubleCountValue); + } else { + this.doubleValue = 0.0; + } + break; + case SUM: + if (doubleCountValue > 0) { + this.doubleValue = doubleValue + (longValue != null ? longValue.doubleValue() : 0.0); + } else { + this.longValue = longValue; + } + break; + case MIN: + case MAX: + if (longCountValue > 0 && doubleCountValue > 0) { + this.doubleValue = MAX.equals(aggType) ? Math.max(doubleValue, longValue.doubleValue()) : Math.min(doubleValue, longValue.doubleValue()); + } else if (doubleCountValue > 0) { + this.doubleValue = doubleValue; + } else if (longCountValue > 0) { + this.longValue = longValue; + } + break; + } + } + } + + public TsKvEntity(Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount) { + if (!isAllNull(booleanValueCount, strValueCount, longValueCount, doubleValueCount)) { + if (booleanValueCount != 0) { + this.longValue = booleanValueCount; + } else if (strValueCount != 0) { + this.longValue = strValueCount; + } else { + this.longValue = longValueCount + doubleValueCount; + } + } + } + + @Override + public boolean isNotEmpty() { + return strValue != null || longValue != null || doubleValue != null || booleanValue != null; + } + + @Override + public TsKvEntry toData() { + KvEntry kvEntry = null; + if (strValue != null) { + kvEntry = new StringDataEntry(strKey, strValue); + } else if (longValue != null) { + kvEntry = new LongDataEntry(strKey, longValue); + } else if (doubleValue != null) { + kvEntry = new DoubleDataEntry(strKey, doubleValue); + } else if (booleanValue != null) { + kvEntry = new BooleanDataEntry(strKey, booleanValue); + } + return new BasicTsKvEntry(ts, kvEntry); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvCompositeKey.java index 49db8554c9..8209b4a77f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvCompositeKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvCompositeKey.java @@ -21,6 +21,7 @@ import lombok.NoArgsConstructor; import javax.persistence.Transient; import java.io.Serializable; +import java.util.UUID; @Data @AllArgsConstructor @@ -30,8 +31,8 @@ public class TimescaleTsKvCompositeKey implements Serializable { @Transient private static final long serialVersionUID = -4089175869616037523L; - private String tenantId; - private String entityId; - private String key; + private UUID tenantId; + private UUID entityId; + private int key; private long ts; } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvEntity.java index a02f030701..626b1104f4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/TimescaleTsKvEntity.java @@ -19,6 +19,11 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ToData; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; @@ -34,9 +39,12 @@ import javax.persistence.NamedNativeQuery; import javax.persistence.SqlResultSetMapping; import javax.persistence.SqlResultSetMappings; import javax.persistence.Table; +import javax.persistence.Transient; +import java.util.UUID; +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.TENANT_ID_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG; import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG_QUERY; import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_COUNT; @@ -118,21 +126,30 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToData { @Id - @Column(name = TENANT_ID_COLUMN) - private String tenantId; + @Column(name = TENANT_ID_COLUMN, columnDefinition = "uuid") + private UUID tenantId; @Id - @Column(name = TS_COLUMN) - protected Long ts; + @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid") + protected UUID entityId; - public TimescaleTsKvEntity() { } + @Id + @Column(name = KEY_COLUMN) + protected int key; + + @Transient + protected String strKey; + + + public TimescaleTsKvEntity() { + } public TimescaleTsKvEntity(Long tsBucket, Long interval, Long longValue, Double doubleValue, Long longCountValue, Long doubleCountValue, String strValue, String aggType) { if (!StringUtils.isEmpty(strValue)) { this.strValue = strValue; } if (!isAllNull(tsBucket, interval, longValue, doubleValue, longCountValue, doubleCountValue)) { - this.ts = tsBucket + interval/2; + this.ts = tsBucket + interval / 2; switch (aggType) { case AVG: double sum = 0.0; @@ -172,7 +189,7 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToD public TimescaleTsKvEntity(Long tsBucket, Long interval, Long booleanValueCount, Long strValueCount, Long longValueCount, Long doubleValueCount) { if (!isAllNull(tsBucket, interval, booleanValueCount, strValueCount, longValueCount, doubleValueCount)) { - this.ts = tsBucket + interval/2; + this.ts = tsBucket + interval / 2; if (booleanValueCount != 0) { this.longValue = booleanValueCount; } else if (strValueCount != 0) { @@ -190,6 +207,17 @@ public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToD @Override public TsKvEntry toData() { - return new BasicTsKvEntry(ts, getKvEntry()); + KvEntry kvEntry = null; + if (strValue != null) { + kvEntry = new StringDataEntry(strKey, strValue); + } else if (longValue != null) { + kvEntry = new LongDataEntry(strKey, longValue); + } else if (doubleValue != null) { + kvEntry = new DoubleDataEntry(strKey, doubleValue); + } else if (booleanValue != null) { + kvEntry = new BooleanDataEntry(strKey, booleanValue); + } + return new BasicTsKvEntry(ts, kvEntry); } + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java index 1d87674231..dd55137f81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java @@ -15,13 +15,8 @@ */ package org.thingsboard.server.dao.sql; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.PreDestroy; -import java.util.concurrent.Executors; - public abstract class JpaAbstractDaoListeningExecutorService { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java index f299717bf2..7c40ad8421 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java @@ -21,8 +21,6 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.support.TransactionTemplate; -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; import java.util.regex.Pattern; @Repository @@ -31,64 +29,15 @@ public abstract class AbstractInsertRepository { private static final ThreadLocal PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); private static final String EMPTY_STR = ""; - protected static final String BOOL_V = "bool_v"; - protected static final String STR_V = "str_v"; - protected static final String LONG_V = "long_v"; - protected static final String DBL_V = "dbl_v"; - - protected static final String TS_KV_LATEST_TABLE = "ts_kv_latest"; - protected static final String TS_KV_TABLE = "ts_kv"; - - protected static final String HSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, BOOL_V); - protected static final String HSQL_ON_STR_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, STR_V); - protected static final String HSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, LONG_V); - protected static final String HSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_TABLE, DBL_V); - - protected static final String HSQL_LATEST_ON_BOOL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, BOOL_V); - protected static final String HSQL_LATEST_ON_STR_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, STR_V); - protected static final String HSQL_LATEST_ON_LONG_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, LONG_V); - protected static final String HSQL_LATEST_ON_DBL_VALUE_UPDATE_SET_NULLS = getHsqlNullValues(TS_KV_LATEST_TABLE, DBL_V); - - protected static final String PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, dbl_v = null"; - protected static final String PSQL_ON_STR_VALUE_UPDATE_SET_NULLS = "bool_v = null, long_v = null, dbl_v = null"; - protected static final String PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = "str_v = null, bool_v = null, dbl_v = null"; - protected static final String PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, bool_v = null"; - @Value("${sql.remove_null_chars}") private boolean removeNullChars; - @PersistenceContext - protected EntityManager entityManager; - @Autowired protected JdbcTemplate jdbcTemplate; @Autowired protected TransactionTemplate transactionTemplate; - protected static String getInsertOrUpdateStringHsql(String tableName, String constraint, String value, String nullValues) { - return "MERGE INTO " + tableName + " USING(VALUES :entity_type, :entity_id, :key, :ts, :" + value + ") A (entity_type, entity_id, key, ts, " + value + ") ON " + constraint + " WHEN MATCHED THEN UPDATE SET " + tableName + "." + value + " = A." + value + ", " + tableName + ".ts = A.ts," + nullValues + "WHEN NOT MATCHED THEN INSERT (entity_type, entity_id, key, ts, " + value + ") VALUES (A.entity_type, A.entity_id, A.key, A.ts, A." + value + ")"; - } - - protected static String getInsertOrUpdateStringPsql(String tableName, String constraint, String value, String nullValues) { - return "INSERT INTO " + tableName + " (entity_type, entity_id, key, ts, " + value + ") VALUES (:entity_type, :entity_id, :key, :ts, :" + value + ") ON CONFLICT " + constraint + " DO UPDATE SET " + value + " = :" + value + ", ts = :ts," + nullValues; - } - - private static String getHsqlNullValues(String tableName, String notNullValue) { - switch (notNullValue) { - case BOOL_V: - return " " + tableName + ".str_v = null, " + tableName + ".long_v = null, " + tableName + ".dbl_v = null "; - case STR_V: - return " " + tableName + ".bool_v = null, " + tableName + ".long_v = null, " + tableName + ".dbl_v = null "; - case LONG_V: - return " " + tableName + ".str_v = null, " + tableName + ".bool_v = null, " + tableName + ".dbl_v = null "; - case DBL_V: - return " " + tableName + ".str_v = null, " + tableName + ".long_v = null, " + tableName + ".bool_v = null "; - default: - throw new RuntimeException("Unsupported insert value: [" + notNullValue + "]"); - } - } - protected String replaceNullChars(String strValue) { if (removeNullChars && strValue != null) { return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractLatestInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractLatestInsertRepository.java deleted file mode 100644 index ff984a43ed..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractLatestInsertRepository.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts; - -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.stereotype.Repository; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; - -import java.util.List; - -@Repository -public abstract class AbstractLatestInsertRepository extends AbstractInsertRepository { - - public abstract void saveOrUpdate(TsKvLatestEntity entity); - - public abstract void saveOrUpdate(List entities); - - protected void processSaveOrUpdate(TsKvLatestEntity entity, String requestBoolValue, String requestStrValue, String requestLongValue, String requestDblValue) { - if (entity.getBooleanValue() != null) { - saveOrUpdateBoolean(entity, requestBoolValue); - } - if (entity.getStrValue() != null) { - saveOrUpdateString(entity, requestStrValue); - } - if (entity.getLongValue() != null) { - saveOrUpdateLong(entity, requestLongValue); - } - if (entity.getDoubleValue() != null) { - saveOrUpdateDouble(entity, requestDblValue); - } - } - - @Modifying - protected abstract void saveOrUpdateBoolean(TsKvLatestEntity entity, String query); - - @Modifying - protected abstract void saveOrUpdateString(TsKvLatestEntity entity, String query); - - @Modifying - protected abstract void saveOrUpdateLong(TsKvLatestEntity entity, String query); - - @Modifying - protected abstract void saveOrUpdateDouble(TsKvLatestEntity entity, String query); - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSimpleSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSimpleSqlTimeseriesDao.java new file mode 100644 index 0000000000..a26eccbf06 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSimpleSqlTimeseriesDao.java @@ -0,0 +1,156 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +@Slf4j +public abstract class AbstractSimpleSqlTimeseriesDao extends AbstractSqlTimeseriesDao { + + @Autowired + private InsertTsRepository insertRepository; + + @Value("${sql.ts.batch_size:1000}") + private int tsBatchSize; + + @Value("${sql.ts.batch_max_delay:100}") + private long tsMaxDelay; + + @Value("${sql.ts.stats_print_interval_ms:1000}") + private long tsStatsPrintIntervalMs; + + protected TbSqlBlockingQueue> tsQueue; + + @PostConstruct + protected void init() { + super.init(); + TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() + .logName("TS") + .batchSize(tsBatchSize) + .maxDelay(tsMaxDelay) + .statsPrintIntervalMs(tsStatsPrintIntervalMs) + .build(); + tsQueue = new TbSqlBlockingQueue<>(tsParams); + tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v)); + } + + @PreDestroy + protected void destroy() { + super.init(); + if (tsQueue != null) { + tsQueue.destroy(); + } + } + + protected ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + if (query.getAggregation() == Aggregation.NONE) { + return findAllAsyncWithLimit(entityId, query); + } else { + long stepTs = query.getStartTs(); + List>> futures = new ArrayList<>(); + while (stepTs < query.getEndTs()) { + long startTs = stepTs; + long endTs = stepTs + query.getInterval(); + long ts = startTs + (endTs - startTs) / 2; + futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); + stepTs = endTs; + } + return getTskvEntriesFuture(Futures.allAsList(futures)); + } + } + + protected abstract ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation); + + protected abstract ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query); + + protected SettableFuture setFutures(List> entitiesFutures) { + SettableFuture listenableFuture = SettableFuture.create(); + CompletableFuture> entities = + CompletableFuture.allOf(entitiesFutures.toArray(new CompletableFuture[entitiesFutures.size()])) + .thenApply(v -> entitiesFutures.stream() + .map(CompletableFuture::join) + .collect(Collectors.toList())); + + entities.whenComplete((tsKvEntities, throwable) -> { + if (throwable != null) { + listenableFuture.setException(throwable); + } else { + T result = null; + for (T entity : tsKvEntities) { + if (entity.isNotEmpty()) { + result = entity; + break; + } + } + listenableFuture.set(result); + } + }); + return listenableFuture; + } + + protected void switchAgregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List> entitiesFutures) { + switch (aggregation) { + case AVG: + findAvg(entityId, key, startTs, endTs, entitiesFutures); + break; + case MAX: + findMax(entityId, key, startTs, endTs, entitiesFutures); + break; + case MIN: + findMin(entityId, key, startTs, endTs, entitiesFutures); + break; + case SUM: + findSum(entityId, key, startTs, endTs, entitiesFutures); + break; + case COUNT: + findCount(entityId, key, startTs, endTs, entitiesFutures); + break; + default: + throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); + } + } + + protected abstract void findCount(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures); + + protected abstract void findSum(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures); + + protected abstract void findMin(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures); + + protected abstract void findMax(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures); + + protected abstract void findAvg(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures); +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 45f477a448..7f340cc48f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -16,20 +16,32 @@ package org.thingsboard.server.dao.sqlts; import com.google.common.base.Function; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.timeseries.TsInsertExecutorType; +import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; +import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import javax.annotation.Nullable; import javax.annotation.PostConstruct; @@ -37,13 +49,55 @@ import javax.annotation.PreDestroy; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.Executors; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; + +@Slf4j public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { private static final String DESC_ORDER = "DESC"; + @Autowired + private TsKvLatestRepository tsKvLatestRepository; + + @Autowired + private InsertLatestRepository insertLatestRepository; + + @Autowired + protected ScheduledLogExecutorComponent logExecutor; + + @Value("${sql.ts_latest.batch_size:1000}") + private int tsLatestBatchSize; + + @Value("${sql.ts_latest.batch_max_delay:100}") + private long tsLatestMaxDelay; + + @Value("${sql.ts_latest.stats_print_interval_ms:1000}") + private long tsLatestStatsPrintIntervalMs; + + private TbSqlBlockingQueue tsLatestQueue; + + @PostConstruct + protected void init() { + TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() + .logName("TS Latest") + .batchSize(tsLatestBatchSize) + .maxDelay(tsLatestMaxDelay) + .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) + .build(); + tsLatestQueue = new TbSqlBlockingQueue<>(tsLatestParams); + tsLatestQueue.init(logExecutor, v -> insertLatestRepository.saveOrUpdate(v)); + } + + @PreDestroy + protected void destroy() { + if (tsLatestQueue != null) { + tsLatestQueue.destroy(); + } + } + protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries .stream() @@ -89,4 +143,105 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx Aggregation.NONE, DESC_ORDER); return findAllAsync(tenantId, entityId, findNewLatestQuery); } + + protected ListenableFuture getFindLatestFuture(EntityId entityId, String key) { + TsKvLatestCompositeKey compositeKey = + new TsKvLatestCompositeKey( + entityId.getEntityType(), + fromTimeUUID(entityId.getId()), + key); + Optional entry = tsKvLatestRepository.findById(compositeKey); + TsKvEntry result; + if (entry.isPresent()) { + result = DaoUtil.getData(entry.get()); + } else { + result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); + } + return Futures.immediateFuture(result); + } + + protected ListenableFuture getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture latestFuture = getFindLatestFuture(entityId, query.getKey()); + + ListenableFuture booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { + long ts = tsKvEntry.getTs(); + return ts > query.getStartTs() && ts <= query.getEndTs(); + }, service); + + ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityType(entityId.getEntityType()); + latestEntity.setEntityId(fromTimeUUID(entityId.getId())); + latestEntity.setKey(query.getKey()); + return service.submit(() -> { + tsKvLatestRepository.delete(latestEntity); + return null; + }); + } + return Futures.immediateFuture(null); + }, service); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + Futures.addCallback(removedLatestFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + if (query.getRewriteLatestIfDeleted()) { + ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + return getNewLatestEntryFuture(tenantId, entityId, query); + } + return Futures.immediateFuture(null); + }, service); + + try { + resultFuture.set(savedLatestFuture.get()); + } catch (InterruptedException | ExecutionException e) { + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); + } + } else { + resultFuture.set(null); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to process remove of the latest value", entityId, t); + } + }); + return resultFuture; + } + + protected ListenableFuture> getFindAllLatestFuture(EntityId entityId) { + return Futures.immediateFuture( + DaoUtil.convertDataList(Lists.newArrayList( + tsKvLatestRepository.findAllByEntityTypeAndEntityId( + entityId.getEntityType(), + UUIDConverter.fromTimeUUID(entityId.getId()))))); + } + + protected ListenableFuture getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityType(entityId.getEntityType()); + latestEntity.setEntityId(fromTimeUUID(entityId.getId())); + latestEntity.setTs(tsKvEntry.getTs()); + latestEntity.setKey(tsKvEntry.getKey()); + latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); + latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); + latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); + latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + return tsLatestQueue.add(latestEntity); + } + + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); + return Futures.transformAsync(future, entryList -> { + if (entryList.size() == 1) { + return getSaveLatestFuture(entityId, entryList.get(0)); + } else { + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); + } + return Futures.immediateFuture(null); + }, service); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractTimeseriesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractTimeseriesInsertRepository.java deleted file mode 100644 index 8a387522f8..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractTimeseriesInsertRepository.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts; - -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.stereotype.Repository; -import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; - -import java.util.List; - -@Repository -public abstract class AbstractTimeseriesInsertRepository extends AbstractInsertRepository { - - public abstract void saveOrUpdate(T entity); - - public abstract void saveOrUpdate(List entities); - - protected void processSaveOrUpdate(T entity, String requestBoolValue, String requestStrValue, String requestLongValue, String requestDblValue) { - if (entity.getBooleanValue() != null) { - saveOrUpdateBoolean(entity, requestBoolValue); - } - if (entity.getStrValue() != null) { - saveOrUpdateString(entity, requestStrValue); - } - if (entity.getLongValue() != null) { - saveOrUpdateLong(entity, requestLongValue); - } - if (entity.getDoubleValue() != null) { - saveOrUpdateDouble(entity, requestDblValue); - } - } - - @Modifying - protected abstract void saveOrUpdateBoolean(T entity, String query); - - @Modifying - protected abstract void saveOrUpdateString(T entity, String query); - - @Modifying - protected abstract void saveOrUpdateLong(T entity, String query); - - @Modifying - protected abstract void saveOrUpdateDouble(T entity, String query); - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/EntityContainer.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/EntityContainer.java new file mode 100644 index 0000000000..3422f34a53 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/EntityContainer.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; + +@Data +@AllArgsConstructor +public class EntityContainer { + + private T entity; + private String partitionDate; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertLatestRepository.java new file mode 100644 index 0000000000..1e1aede157 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertLatestRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; + +import java.util.List; + +public interface InsertLatestRepository { + + void saveOrUpdate(List entities); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertTsRepository.java new file mode 100644 index 0000000000..6ab11618f0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/InsertTsRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; + +import java.util.List; + +public interface InsertTsRepository { + + void saveOrUpdate(List> entities); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java new file mode 100644 index 0000000000..60284c73cf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.dictionary; + +import org.springframework.data.repository.CrudRepository; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.util.PsqlDao; + +import java.util.Optional; + +@PsqlDao +public interface TsKvDictionaryRepository extends CrudRepository { + + Optional findByKeyId(int keyId); + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/HsqlTimeseriesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/HsqlTimeseriesInsertRepository.java new file mode 100644 index 0000000000..5cc344aa2a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/HsqlTimeseriesInsertRepository.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.hsql; + +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity; +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; +import org.thingsboard.server.dao.sqlts.EntityContainer; +import org.thingsboard.server.dao.sqlts.InsertTsRepository; +import org.thingsboard.server.dao.util.HsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; + +@SqlTsDao +@HsqlDao +@Repository +@Transactional +public class HsqlTimeseriesInsertRepository extends AbstractInsertRepository implements InsertTsRepository { + + private static final String INSERT_OR_UPDATE = + "MERGE INTO ts_kv USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " + + "T (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " + + "ON (ts_kv.entity_type=T.entity_type " + + "AND ts_kv.entity_id=T.entity_id " + + "AND ts_kv.key=T.key " + + "AND ts_kv.ts=T.ts) " + + "WHEN MATCHED THEN UPDATE SET ts_kv.bool_v = T.bool_v, ts_kv.str_v = T.str_v, ts_kv.long_v = T.long_v, ts_kv.dbl_v = T.dbl_v " + + "WHEN NOT MATCHED THEN INSERT (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " + + "VALUES (T.entity_type, T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);"; + + @Override + public void saveOrUpdate(List> entities) { + jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + EntityContainer tsKvEntityEntityContainer = entities.get(i); + TsKvEntity tsKvEntity = tsKvEntityEntityContainer.getEntity(); + ps.setString(1, tsKvEntity.getEntityType().name()); + ps.setString(2, tsKvEntity.getEntityId()); + ps.setString(3, tsKvEntity.getKey()); + ps.setLong(4, tsKvEntity.getTs()); + + if (tsKvEntity.getBooleanValue() != null) { + ps.setBoolean(5, tsKvEntity.getBooleanValue()); + } else { + ps.setNull(5, Types.BOOLEAN); + } + + ps.setString(6, tsKvEntity.getStrValue()); + + if (tsKvEntity.getLongValue() != null) { + ps.setLong(7, tsKvEntity.getLongValue()); + } else { + ps.setNull(7, Types.BIGINT); + } + + if (tsKvEntity.getDoubleValue() != null) { + ps.setDouble(8, tsKvEntity.getDoubleValue()); + } else { + ps.setNull(8, Types.DOUBLE); + } + } + + @Override + public int getBatchSize() { + return entities.size(); + } + }); + } +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java new file mode 100644 index 0000000000..c8bafe81e0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java @@ -0,0 +1,206 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.hsql; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity; +import org.thingsboard.server.dao.sqlts.AbstractSimpleSqlTimeseriesDao; +import org.thingsboard.server.dao.sqlts.EntityContainer; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.dao.util.HsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; + + +@Component +@Slf4j +@SqlTsDao +@HsqlDao +public class JpaHsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao implements TimeseriesDao { + + @Autowired + private TsKvHsqlRepository tsKvRepository; + + @Override + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + return processFindAllAsync(tenantId, entityId, queries); + } + + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + TsKvEntity entity = new TsKvEntity(); + entity.setEntityType(entityId.getEntityType()); + entity.setEntityId(fromTimeUUID(entityId.getId())); + entity.setTs(tsKvEntry.getTs()); + entity.setKey(tsKvEntry.getKey()); + entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); + entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + log.trace("Saving entity: {}", entity); + return tsQueue.add(new EntityContainer(entity, null)); + } + + @Override + public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return service.submit(() -> { + tsKvRepository.delete( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + query.getKey(), + query.getStartTs(), + query.getEndTs()); + return null; + }); + } + + @Override + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + return getSaveLatestFuture(entityId, tsKvEntry); + } + + @Override + public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return getRemoveLatestFuture(tenantId, entityId, query); + } + + @Override + public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { + return getFindLatestFuture(entityId, key); + } + + @Override + public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { + return getFindAllLatestFuture(entityId); + } + + @Override + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + return Futures.immediateFuture(null); + } + + @Override + public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return Futures.immediateFuture(null); + } + + protected ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + List> entitiesFutures = new ArrayList<>(); + switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); + return Futures.transform(setFutures(entitiesFutures), entity -> { + if (entity != null && entity.isNotEmpty()) { + entity.setEntityId(fromTimeUUID(entityId.getId())); + entity.setEntityType(entityId.getEntityType()); + entity.setKey(key); + entity.setTs(ts); + return Optional.of(DaoUtil.getData(entity)); + } else { + return Optional.empty(); + } + }); + } + + protected ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + return Futures.immediateFuture( + DaoUtil.convertDataList( + tsKvRepository.findAllWithLimit( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + query.getKey(), + query.getStartTs(), + query.getEndTs(), + new PageRequest(0, query.getLimit(), + new Sort(Sort.Direction.fromString( + query.getOrderBy()), "ts"))))); + } + + protected void findCount(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + entitiesFutures.add(tsKvRepository.findCount( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + key, + startTs, + endTs)); + } + + protected void findSum(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + entitiesFutures.add(tsKvRepository.findSum( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + key, + startTs, + endTs)); + } + + protected void findMin(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + entitiesFutures.add(tsKvRepository.findStringMin( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + key, + startTs, + endTs)); + entitiesFutures.add(tsKvRepository.findNumericMin( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + key, + startTs, + endTs)); + } + + protected void findMax(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + entitiesFutures.add(tsKvRepository.findStringMax( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + key, + startTs, + endTs)); + entitiesFutures.add(tsKvRepository.findNumericMax( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + key, + startTs, + endTs)); + } + + protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + entitiesFutures.add(tsKvRepository.findAvg( + fromTimeUUID(entityId.getId()), + entityId.getEntityType(), + key, + startTs, + endTs)); + } + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/TsKvHsqlRepository.java similarity index 96% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/TsKvHsqlRepository.java index 64a3b43574..f770b4dca9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/TsKvHsqlRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sqlts.ts; +package org.thingsboard.server.dao.sqlts.hsql; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.Modifying; @@ -23,15 +23,15 @@ import org.springframework.data.repository.query.Param; import org.springframework.scheduling.annotation.Async; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvCompositeKey; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvCompositeKey; +import org.thingsboard.server.dao.model.sqlts.hsql.TsKvEntity; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.concurrent.CompletableFuture; @SqlDao -public interface TsKvRepository extends CrudRepository { +public interface TsKvHsqlRepository extends CrudRepository { @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + "AND tskv.entityType = :entityType AND tskv.key = :entityKey " + @@ -146,4 +146,4 @@ public interface TsKvRepository extends CrudRepository entities) { + jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + ps.setString(1, entities.get(i).getEntityType().name()); + ps.setString(2, entities.get(i).getEntityId()); + ps.setString(3, entities.get(i).getKey()); + ps.setLong(4, entities.get(i).getTs()); + + if (entities.get(i).getBooleanValue() != null) { + ps.setBoolean(5, entities.get(i).getBooleanValue()); + } else { + ps.setNull(5, Types.BOOLEAN); + } + + ps.setString(6, entities.get(i).getStrValue()); + + if (entities.get(i).getLongValue() != null) { + ps.setLong(7, entities.get(i).getLongValue()); + } else { + ps.setNull(7, Types.BIGINT); + } + + if (entities.get(i).getDoubleValue() != null) { + ps.setDouble(8, entities.get(i).getDoubleValue()); + } else { + ps.setNull(8, Types.DOUBLE); + } + } + + @Override + public int getBatchSize() { + return entities.size(); + } + }); + } +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/PsqlLatestInsertRepository.java similarity index 64% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/PsqlLatestInsertRepository.java index 16c0bb5426..bace8ff637 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/PsqlLatestInsertRepository.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sqlts.ts; +package org.thingsboard.server.dao.sqlts.latest; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.stereotype.Repository; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository; -import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; +import org.thingsboard.server.dao.sqlts.InsertLatestRepository; +import org.thingsboard.server.dao.util.PsqlTsAnyDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -31,18 +31,11 @@ import java.sql.Types; import java.util.ArrayList; import java.util.List; -@SqlTsDao -@PsqlDao + +@PsqlTsAnyDao @Repository @Transactional -public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository { - - private static final String TS_KV_LATEST_CONSTRAINT = "(entity_type, entity_id, key)"; - - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); +public class PsqlLatestInsertRepository extends AbstractInsertRepository implements InsertLatestRepository { private static final String BATCH_UPDATE = "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ? WHERE entity_type = ? AND entity_id = ? and key = ?"; @@ -52,11 +45,6 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository { "INSERT INTO ts_kv_latest (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT (entity_type, entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; - @Override - public void saveOrUpdate(TsKvLatestEntity entity) { - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); - } - @Override public void saveOrUpdate(List entities) { transactionTemplate.execute(new TransactionCallbackWithoutResult() { @@ -160,48 +148,4 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository { } }); } - - @Override - protected void saveOrUpdateBoolean(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("bool_v", entity.getBooleanValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateString(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("str_v", replaceNullChars(entity.getStrValue())) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateLong(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("long_v", entity.getLongValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateDouble(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("dbl_v", entity.getDoubleValue()) - .executeUpdate(); - } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java similarity index 83% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvLatestRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index ecefc2a86e..71c8a00057 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sqlts.ts; +package org.thingsboard.server.dao.sqlts.latest; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestCompositeKey; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java new file mode 100644 index 0000000000..fbaa2f9acf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java @@ -0,0 +1,312 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.psql; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.hibernate.exception.ConstraintViolationException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity; +import org.thingsboard.server.dao.sqlts.AbstractSimpleSqlTimeseriesDao; +import org.thingsboard.server.dao.sqlts.EntityContainer; +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import org.thingsboard.server.dao.timeseries.PsqlPartition; +import org.thingsboard.server.dao.timeseries.SqlTsPartitionDate; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; + +import static org.thingsboard.server.dao.timeseries.SqlTsPartitionDate.EPOCH_START; + + +@Component +@Slf4j +@SqlTsDao +@PsqlDao +public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao implements TimeseriesDao { + + private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); + private final Set partitions = ConcurrentHashMap.newKeySet(); + + private static final ReentrantLock tsCreationLock = new ReentrantLock(); + private static final ReentrantLock partitionCreationLock = new ReentrantLock(); + + @Autowired + private TsKvDictionaryRepository dictionaryRepository; + + @Autowired + private TsKvPsqlRepository tsKvRepository; + + @Autowired + private PsqlPartitioningRepository partitioningRepository; + + private SqlTsPartitionDate tsFormat; + + @Value("${sql.ts_key_value_partitioning}") + private String partitioning; + + @Override + protected void init() { + super.init(); + Optional partition = SqlTsPartitionDate.parse(partitioning); + if (partition.isPresent()) { + tsFormat = partition.get(); + } else { + log.warn("Incorrect configuration of partitioning {}", partitioning); + throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!"); + } + } + + @Override + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + return processFindAllAsync(tenantId, entityId, queries); + } + + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + String strKey = tsKvEntry.getKey(); + Integer keyId = getOrSaveKeyId(strKey); + TsKvEntity entity = new TsKvEntity(); + entity.setEntityId(entityId.getId()); + entity.setTs(tsKvEntry.getTs()); + entity.setKey(keyId); + entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); + entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + PsqlPartition psqlPartition = toPartition(tsKvEntry.getTs()); + savePartition(psqlPartition); + log.trace("Saving entity: {}", entity); + return tsQueue.add(new EntityContainer(entity, psqlPartition.getPartitionDate())); + } + + @Override + public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return service.submit(() -> { + String strKey = query.getKey(); + Integer keyId = getOrSaveKeyId(strKey); + tsKvRepository.delete( + entityId.getId(), + keyId, + query.getStartTs(), + query.getEndTs()); + return null; + }); + } + + @Override + public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return getRemoveLatestFuture(tenantId, entityId, query); + } + + @Override + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + return getSaveLatestFuture(entityId, tsKvEntry); + } + + @Override + public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { + return getFindLatestFuture(entityId, key); + } + + @Override + public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { + return getFindAllLatestFuture(entityId); + } + + @Override + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + return Futures.immediateFuture(null); + } + + @Override + public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return Futures.immediateFuture(null); + } + + protected ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + List> entitiesFutures = new ArrayList<>(); + switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); + return Futures.transform(setFutures(entitiesFutures), entity -> { + if (entity != null && entity.isNotEmpty()) { + entity.setEntityId(entityId.getId()); + entity.setStrKey(key); + entity.setTs(ts); + return Optional.of(DaoUtil.getData(entity)); + } else { + return Optional.empty(); + } + }); + } + + protected ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + Integer keyId = getOrSaveKeyId(query.getKey()); + List tsKvEntities = tsKvRepository.findAllWithLimit( + entityId.getId(), + keyId, + query.getStartTs(), + query.getEndTs(), + new PageRequest(0, query.getLimit(), + new Sort(Sort.Direction.fromString( + query.getOrderBy()), "ts"))); + tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); + return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); + } + + protected void findCount(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + Integer keyId = getOrSaveKeyId(key); + entitiesFutures.add(tsKvRepository.findCount( + entityId.getId(), + keyId, + startTs, + endTs)); + } + + protected void findSum(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + Integer keyId = getOrSaveKeyId(key); + entitiesFutures.add(tsKvRepository.findSum( + entityId.getId(), + keyId, + startTs, + endTs)); + } + + protected void findMin(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + Integer keyId = getOrSaveKeyId(key); + entitiesFutures.add(tsKvRepository.findStringMin( + entityId.getId(), + keyId, + startTs, + endTs)); + entitiesFutures.add(tsKvRepository.findNumericMin( + entityId.getId(), + keyId, + startTs, + endTs)); + } + + protected void findMax(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + Integer keyId = getOrSaveKeyId(key); + entitiesFutures.add(tsKvRepository.findStringMax( + entityId.getId(), + keyId, + startTs, + endTs)); + entitiesFutures.add(tsKvRepository.findNumericMax( + entityId.getId(), + keyId, + startTs, + endTs)); + } + + protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures) { + Integer keyId = getOrSaveKeyId(key); + entitiesFutures.add(tsKvRepository.findAvg( + entityId.getId(), + keyId, + startTs, + endTs)); + } + + private Integer getOrSaveKeyId(String strKey) { + Integer keyId = tsKvDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + tsCreationLock.lock(); + try { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + TsKvDictionary tsKvDictionary = new TsKvDictionary(); + tsKvDictionary.setKey(strKey); + try { + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (ConstraintViolationException e) { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + tsCreationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + tsKvDictionaryMap.put(strKey, keyId); + } + } + return keyId; + } + + private void savePartition(PsqlPartition psqlPartition) { + if (!partitions.contains(psqlPartition)) { + partitionCreationLock.lock(); + try { + log.trace("Saving partition: {}", psqlPartition); + partitioningRepository.save(psqlPartition); + log.trace("Adding partition to Set: {}", psqlPartition); + partitions.add(psqlPartition); + } finally { + partitionCreationLock.unlock(); + } + } + } + + private PsqlPartition toPartition(long ts) { + LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); + LocalDateTime localDateTimeStart = tsFormat.trancateTo(time); + if (localDateTimeStart == SqlTsPartitionDate.EPOCH_START) { + return new PsqlPartition(toMills(EPOCH_START), Long.MAX_VALUE, tsFormat.getPattern()); + } else { + LocalDateTime localDateTimeEnd = tsFormat.plusTo(localDateTimeStart); + return new PsqlPartition(toMills(localDateTimeStart), toMills(localDateTimeEnd), tsFormat.getPattern()); + } + } + + private long toMills(LocalDateTime time) { return time.toInstant(ZoneOffset.UTC).toEpochMilli(); } +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlPartitioningRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlPartitioningRepository.java new file mode 100644 index 0000000000..0e22cb26ba --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlPartitioningRepository.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.psql; + +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.dao.timeseries.PsqlPartition; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; + +@SqlTsDao +@PsqlDao +@Repository +@Transactional +public class PsqlPartitioningRepository { + + @PersistenceContext + private EntityManager entityManager; + + public void save(PsqlPartition partition) { + entityManager.createNativeQuery(partition.getQuery()) + .executeUpdate(); + } + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlTimeseriesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlTimeseriesInsertRepository.java new file mode 100644 index 0000000000..b1aaff4ec8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/PsqlTimeseriesInsertRepository.java @@ -0,0 +1,101 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.psql; + +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity; +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; +import org.thingsboard.server.dao.sqlts.EntityContainer; +import org.thingsboard.server.dao.sqlts.InsertTsRepository; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@SqlTsDao +@PsqlDao +@Repository +@Transactional +public class PsqlTimeseriesInsertRepository extends AbstractInsertRepository implements InsertTsRepository { + + private static final String INSERT_INTO_TS_KV = "INSERT INTO ts_kv_"; + + private static final String VALUES_ON_CONFLICT_DO_UPDATE = " (entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES (?, ?, ?, ?, ?, ?, ?) " + + "ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; + + @Override + public void saveOrUpdate(List> entities) { + Map> partitionMap = new HashMap<>(); + for (EntityContainer entityContainer : entities) { + List tsKvEntities = partitionMap.computeIfAbsent(entityContainer.getPartitionDate(), k -> new ArrayList<>()); + tsKvEntities.add(entityContainer.getEntity()); + } + partitionMap.forEach((partition, entries) -> jdbcTemplate.batchUpdate(getInsertOrUpdateQuery(partition), new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + TsKvEntity tsKvEntity = entries.get(i); + ps.setObject(1, tsKvEntity.getEntityId()); + ps.setInt(2, tsKvEntity.getKey()); + ps.setLong(3, tsKvEntity.getTs()); + + if (tsKvEntity.getBooleanValue() != null) { + ps.setBoolean(4, tsKvEntity.getBooleanValue()); + ps.setBoolean(8, tsKvEntity.getBooleanValue()); + } else { + ps.setNull(4, Types.BOOLEAN); + ps.setNull(8, Types.BOOLEAN); + } + + ps.setString(5, replaceNullChars(tsKvEntity.getStrValue())); + ps.setString(9, replaceNullChars(tsKvEntity.getStrValue())); + + + if (tsKvEntity.getLongValue() != null) { + ps.setLong(6, tsKvEntity.getLongValue()); + ps.setLong(10, tsKvEntity.getLongValue()); + } else { + ps.setNull(6, Types.BIGINT); + ps.setNull(10, Types.BIGINT); + } + + if (tsKvEntity.getDoubleValue() != null) { + ps.setDouble(7, tsKvEntity.getDoubleValue()); + ps.setDouble(11, tsKvEntity.getDoubleValue()); + } else { + ps.setNull(7, Types.DOUBLE); + ps.setNull(11, Types.DOUBLE); + } + } + + @Override + public int getBatchSize() { + return entries.size(); + } + })); + } + + private String getInsertOrUpdateQuery(String partitionDate) { + return INSERT_INTO_TS_KV + partitionDate + VALUES_ON_CONFLICT_DO_UPDATE; + } +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/TsKvPsqlRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/TsKvPsqlRepository.java new file mode 100644 index 0000000000..7b3328f86b --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/TsKvPsqlRepository.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.psql; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.scheduling.annotation.Async; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.dao.model.sqlts.psql.TsKvCompositeKey; +import org.thingsboard.server.dao.model.sqlts.psql.TsKvEntity; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +@SqlDao +public interface TsKvPsqlRepository extends CrudRepository { + + @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + + "AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + List findAllWithLimit(@Param("entityId") UUID entityId, + @Param("entityKey") int key, + @Param("startTs") long startTs, + @Param("endTs") long endTs, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + + "AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + void delete(@Param("entityId") UUID entityId, + @Param("entityKey") int key, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + + @Async + @Query("SELECT new TsKvEntity(MAX(tskv.strValue)) FROM TsKvEntity tskv " + + "WHERE tskv.strValue IS NOT NULL " + + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + CompletableFuture findStringMax(@Param("entityId") UUID entityId, + @Param("entityKey") int entityKey, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + + @Async + @Query("SELECT new TsKvEntity(MAX(COALESCE(tskv.longValue, -9223372036854775807)), " + + "MAX(COALESCE(tskv.doubleValue, -1.79769E+308)), " + + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + + "'MAX') FROM TsKvEntity tskv " + + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + CompletableFuture findNumericMax(@Param("entityId") UUID entityId, + @Param("entityKey") int entityKey, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + + + @Async + @Query("SELECT new TsKvEntity(MIN(tskv.strValue)) FROM TsKvEntity tskv " + + "WHERE tskv.strValue IS NOT NULL " + + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + CompletableFuture findStringMin(@Param("entityId") UUID entityId, + @Param("entityKey") int entityKey, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + + @Async + @Query("SELECT new TsKvEntity(MIN(COALESCE(tskv.longValue, 9223372036854775807)), " + + "MIN(COALESCE(tskv.doubleValue, 1.79769E+308)), " + + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + + "'MIN') FROM TsKvEntity tskv " + + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + CompletableFuture findNumericMin( + @Param("entityId") UUID entityId, + @Param("entityKey") int entityKey, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + + @Async + @Query("SELECT new TsKvEntity(SUM(CASE WHEN tskv.booleanValue IS NULL THEN 0 ELSE 1 END), " + + "SUM(CASE WHEN tskv.strValue IS NULL THEN 0 ELSE 1 END), " + + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END)) FROM TsKvEntity tskv " + + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + CompletableFuture findCount(@Param("entityId") UUID entityId, + @Param("entityKey") int entityKey, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + + @Async + @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " + + "SUM(COALESCE(tskv.doubleValue, 0.0)), " + + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + + "'AVG') FROM TsKvEntity tskv " + + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + CompletableFuture findAvg(@Param("entityId") UUID entityId, + @Param("entityKey") int entityKey, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + + @Async + @Query("SELECT new TsKvEntity(SUM(COALESCE(tskv.longValue, 0)), " + + "SUM(COALESCE(tskv.doubleValue, 0.0)), " + + "SUM(CASE WHEN tskv.longValue IS NULL THEN 0 ELSE 1 END), " + + "SUM(CASE WHEN tskv.doubleValue IS NULL THEN 0 ELSE 1 END), " + + "'SUM') FROM TsKvEntity tskv " + + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + CompletableFuture findSum(@Param("entityId") UUID entityId, + @Param("entityKey") int entityKey, + @Param("startTs") long startTs, + @Param("endTs") long endTs); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java index da39086da8..15deb702a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java @@ -23,6 +23,7 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import java.util.List; +import java.util.UUID; import java.util.concurrent.CompletableFuture; @Repository @@ -36,7 +37,7 @@ public class AggregationRepository { public static final String FIND_COUNT = "findCount"; - public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS varchar) AND tskv.entity_id = cast(:entityId AS varchar) AND tskv.key= cast(:entityKey AS varchar) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket"; + public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS uuid) AND tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket"; public static final String FIND_AVG_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts) AS tsBucket, :timeBucket AS interval, SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, null AS strValue, 'AVG' AS aggType "; @@ -52,41 +53,41 @@ public class AggregationRepository { private EntityManager entityManager; @Async - public CompletableFuture> findAvg(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { + public CompletableFuture> findAvg(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { @SuppressWarnings("unchecked") List resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG); return CompletableFuture.supplyAsync(() -> resultList); } @Async - public CompletableFuture> findMax(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { + public CompletableFuture> findMax(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { @SuppressWarnings("unchecked") List resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX); return CompletableFuture.supplyAsync(() -> resultList); } @Async - public CompletableFuture> findMin(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { + public CompletableFuture> findMin(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { @SuppressWarnings("unchecked") List resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN); return CompletableFuture.supplyAsync(() -> resultList); } @Async - public CompletableFuture> findSum(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { + public CompletableFuture> findSum(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { @SuppressWarnings("unchecked") List resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM); return CompletableFuture.supplyAsync(() -> resultList); } @Async - public CompletableFuture> findCount(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs) { + public CompletableFuture> findCount(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) { @SuppressWarnings("unchecked") List resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT); return CompletableFuture.supplyAsync(() -> resultList); } - private List getResultList(String tenantId, String entityId, String entityKey, long timeBucket, long startTs, long endTs, String query) { + private List getResultList(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) { return entityManager.createNamedQuery(query) .setParameter("tenantId", tenantId) .setParameter("entityId", entityId) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java index 2adaa045ac..35f0fd497a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java @@ -19,7 +19,9 @@ import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity; -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; +import org.thingsboard.server.dao.sqlts.AbstractInsertRepository; +import org.thingsboard.server.dao.sqlts.EntityContainer; +import org.thingsboard.server.dao.sqlts.InsertTsRepository; import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; @@ -32,35 +34,21 @@ import java.util.List; @PsqlDao @Repository @Transactional -public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepository { - - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateString(BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateString(STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateString(LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateString(DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); - - private static final String BATCH_UPDATE = - "UPDATE tenant_ts_kv SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ? WHERE entity_type = ? AND entity_id = ? and key = ? and ts = ?"; - +public class TimescaleInsertRepository extends AbstractInsertRepository implements InsertTsRepository { private static final String INSERT_OR_UPDATE = "INSERT INTO tenant_ts_kv (tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; @Override - public void saveOrUpdate(TimescaleTsKvEntity entity) { - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); - } - - @Override - public void saveOrUpdate(List entities) { + public void saveOrUpdate(List> entities) { jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { - TimescaleTsKvEntity tsKvEntity = entities.get(i); - ps.setString(1, tsKvEntity.getTenantId()); - ps.setString(2, tsKvEntity.getEntityId()); - ps.setString(3, tsKvEntity.getKey()); + TimescaleTsKvEntity tsKvEntity = entities.get(i).getEntity(); + ps.setObject(1, tsKvEntity.getTenantId()); + ps.setObject(2, tsKvEntity.getEntityId()); + ps.setInt(3, tsKvEntity.getKey()); ps.setLong(4, tsKvEntity.getTs()); if (tsKvEntity.getBooleanValue() != null) { @@ -98,52 +86,4 @@ public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepositor } }); } - - @Override - protected void saveOrUpdateBoolean(TimescaleTsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("tenant_id", entity.getTenantId()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("bool_v", entity.getBooleanValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateString(TimescaleTsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("tenant_id", entity.getTenantId()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("str_v", replaceNullChars(entity.getStrValue())) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateLong(TimescaleTsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("tenant_id", entity.getTenantId()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("long_v", entity.getLongValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateDouble(TimescaleTsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("tenant_id", entity.getTenantId()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("dbl_v", entity.getDoubleValue()) - .executeUpdate(); - } - - private static String getInsertOrUpdateString(String value, String nullValues) { - return "INSERT INTO tenant_ts_kv(tenant_id, entity_id, key, ts, " + value + ") VALUES (:tenant_id, :entity_id, :key, :ts, :" + value + ") ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET " + value + " = :" + value + ", ts = :ts," + nullValues; - } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 9a2f534a59..6de9e3c51a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -15,11 +15,11 @@ */ package org.thingsboard.server.dao.sqlts.timescale; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; +import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; @@ -29,19 +29,19 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.data.kv.TsKvQuery; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity; -import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; +import org.thingsboard.server.dao.sqlts.EntityContainer; +import org.thingsboard.server.dao.sqlts.InsertTsRepository; +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; @@ -51,9 +51,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CompletableFuture; - -import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; @Component @@ -63,17 +65,21 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements private static final String TS = "ts"; + private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); + + private static final ReentrantLock tsCreationLock = new ReentrantLock(); + @Autowired - private TsKvTimescaleRepository tsKvRepository; + private TsKvDictionaryRepository dictionaryRepository; @Autowired - private AggregationRepository aggregationRepository; + private TsKvTimescaleRepository tsKvRepository; @Autowired - private AbstractTimeseriesInsertRepository insertRepository; + private AggregationRepository aggregationRepository; @Autowired - ScheduledLogExecutorComponent logExecutor; + private InsertTsRepository insertRepository; @Value("${sql.ts_timescale.batch_size:1000}") private int batchSize; @@ -84,10 +90,11 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Value("${sql.ts_timescale.stats_print_interval_ms:1000}") private long statsPrintIntervalMs; - private TbSqlBlockingQueue queue; + private TbSqlBlockingQueue> queue; @PostConstruct - private void init() { + protected void init() { + super.init(); TbSqlBlockingQueueParams params = TbSqlBlockingQueueParams.builder() .logName("TS Timescale") .batchSize(batchSize) @@ -99,17 +106,13 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } @PreDestroy - private void destroy() { + protected void destroy() { + super.init(); if (queue != null) { queue.destroy(); } } - @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { - return processFindAllAsync(tenantId, entityId, queries); - } - protected ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); @@ -122,50 +125,36 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } } - private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - return Futures.immediateFuture( - DaoUtil.convertDataList( - tsKvRepository.findAllWithLimit( - fromTimeUUID(tenantId.getId()), - fromTimeUUID(entityId.getId()), - query.getKey(), - query.getStartTs(), - query.getEndTs(), - new PageRequest(0, query.getLimit(), - new Sort(Sort.Direction.fromString( - query.getOrderBy()), "ts"))))); + @Override + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + return processFindAllAsync(tenantId, entityId, queries); } - @Override public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - ListenableFuture> future = getLatest(tenantId, entityId, key, 0L, System.currentTimeMillis()); - return Futures.transform(future, latest -> { - if (!CollectionUtils.isEmpty(latest)) { - return DaoUtil.getData(latest.get(0)); - } else { - return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); - } - }, service); + return getFindLatestFuture(entityId, key); } @Override public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - return Futures.immediateFuture(DaoUtil.convertDataList(Lists.newArrayList(tsKvRepository.findAllLatestValues(fromTimeUUID(tenantId.getId()), fromTimeUUID(entityId.getId()))))); + return getFindAllLatestFuture(entityId); } @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + String strKey = tsKvEntry.getKey(); + Integer keyId = getOrSaveKeyId(strKey); TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); - entity.setTenantId(fromTimeUUID(tenantId.getId())); - entity.setEntityId(fromTimeUUID(entityId.getId())); + entity.setTenantId(tenantId.getId()); + entity.setEntityId(entityId.getId()); entity.setTs(tsKvEntry.getTs()); - entity.setKey(tsKvEntry.getKey()); + entity.setKey(keyId); entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - return queue.add(entity); + log.trace("Saving entity to timescale db: {}", entity); + return queue.add(new EntityContainer(entity, null)); } @Override @@ -175,16 +164,18 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - return Futures.immediateFuture(null); + return getSaveLatestFuture(entityId, tsKvEntry); } @Override public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + String strKey = query.getKey(); + Integer keyId = getOrSaveKeyId(strKey); return service.submit(() -> { tsKvRepository.delete( - fromTimeUUID(tenantId.getId()), - fromTimeUUID(entityId.getId()), - query.getKey(), + tenantId.getId(), + entityId.getId(), + keyId, query.getStartTs(), query.getEndTs()); return null; @@ -193,7 +184,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Override public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return service.submit(() -> null); + return getRemoveLatestFuture(tenantId, entityId, query); } @Override @@ -201,37 +192,60 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements return service.submit(() -> null); } - private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); - return Futures.transformAsync(future, entryList -> { - if (entryList.size() == 1) { - return save(tenantId, entityId, entryList.get(0), 0L); - } else { - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, service); - } - - private ListenableFuture> findLatestByQuery(TenantId tenantId, EntityId entityId, TsKvQuery query) { - return getLatest(tenantId, entityId, query.getKey(), query.getStartTs(), query.getEndTs()); + private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + String strKey = query.getKey(); + Integer keyId = getOrSaveKeyId(strKey); + List timescaleTsKvEntities = tsKvRepository.findAllWithLimit( + tenantId.getId(), + entityId.getId(), + keyId, + query.getStartTs(), + query.getEndTs(), + new PageRequest(0, query.getLimit(), + new Sort(Sort.Direction.fromString( + query.getOrderBy()), TS))); + timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); + return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities)); } - private ListenableFuture> getLatest(TenantId tenantId, EntityId entityId, String key, long start, long end) { - return Futures.immediateFuture(tsKvRepository.findAllWithLimit( - fromTimeUUID(tenantId.getId()), - fromTimeUUID(entityId.getId()), - key, - start, - end, - new PageRequest(0, 1, - new Sort(Sort.Direction.DESC, TS)))); + private Integer getOrSaveKeyId(String strKey) { + Integer keyId = tsKvDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + tsCreationLock.lock(); + try { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + TsKvDictionary tsKvDictionary = new TsKvDictionary(); + tsKvDictionary.setKey(strKey); + try { + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (ConstraintViolationException e) { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + tsCreationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + tsKvDictionaryMap.put(strKey, keyId); + } + } + return keyId; } private ListenableFuture>> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) { - String entityIdStr = fromTimeUUID(entityId.getId()); - String tenantIdStr = fromTimeUUID(tenantId.getId()); - CompletableFuture> listCompletableFuture = switchAgregation(key, startTs, endTs, timeBucket, aggregation, entityIdStr, tenantIdStr); + CompletableFuture> listCompletableFuture = switchAgregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId(), tenantId.getId()); SettableFuture> listenableFuture = SettableFuture.create(); listCompletableFuture.whenComplete((timescaleTsKvEntities, throwable) -> { if (throwable != null) { @@ -245,9 +259,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements List> result = new ArrayList<>(); timescaleTsKvEntities.forEach(entity -> { if (entity != null && entity.isNotEmpty()) { - entity.setEntityId(entityIdStr); - entity.setTenantId(tenantIdStr); - entity.setKey(key); + entity.setEntityId(entityId.getId()); + entity.setTenantId(tenantId.getId()); + entity.setStrKey(key); result.add(Optional.of(DaoUtil.getData(entity))); } else { result.add(Optional.empty()); @@ -260,69 +274,74 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements }); } - private CompletableFuture> switchAgregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, String entityIdStr, String tenantIdStr) { + private CompletableFuture> switchAgregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId, UUID tenantId) { switch (aggregation) { case AVG: - return findAvg(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); + return findAvg(key, startTs, endTs, timeBucket, entityId, tenantId); case MAX: - return findMax(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); + return findMax(key, startTs, endTs, timeBucket, entityId, tenantId); case MIN: - return findMin(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); + return findMin(key, startTs, endTs, timeBucket, entityId, tenantId); case SUM: - return findSum(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); + return findSum(key, startTs, endTs, timeBucket, entityId, tenantId); case COUNT: - return findCount(key, startTs, endTs, timeBucket, entityIdStr, tenantIdStr); + return findCount(key, startTs, endTs, timeBucket, entityId, tenantId); default: throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); } } - private CompletableFuture> findAvg(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { + private CompletableFuture> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { + Integer keyId = getOrSaveKeyId(key); return aggregationRepository.findAvg( - tenantIdStr, - entityIdStr, - key, + tenantId, + entityId, + keyId, timeBucket, startTs, endTs); } - private CompletableFuture> findMax(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { + private CompletableFuture> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { + Integer keyId = getOrSaveKeyId(key); return aggregationRepository.findMax( - tenantIdStr, - entityIdStr, - key, + tenantId, + entityId, + keyId, timeBucket, startTs, endTs); } - private CompletableFuture> findMin(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { + private CompletableFuture> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { + Integer keyId = getOrSaveKeyId(key); return aggregationRepository.findMin( - tenantIdStr, - entityIdStr, - key, + tenantId, + entityId, + keyId, timeBucket, startTs, endTs); } - private CompletableFuture> findSum(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { + private CompletableFuture> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { + Integer keyId = getOrSaveKeyId(key); return aggregationRepository.findSum( - tenantIdStr, - entityIdStr, - key, + tenantId, + entityId, + keyId, timeBucket, startTs, endTs); } - private CompletableFuture> findCount(String key, long startTs, long endTs, long timeBucket, String entityIdStr, String tenantIdStr) { + private CompletableFuture> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) { + Integer keyId = getOrSaveKeyId(key); return aggregationRepository.findCount( - tenantIdStr, - entityIdStr, - key, + tenantId, + entityId, + keyId, timeBucket, startTs, endTs); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java index 9f68bec921..a4b15abd26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.model.sqlts.timescale.TimescaleTsKvEntity; import org.thingsboard.server.dao.util.TimescaleDBTsDao; import java.util.List; +import java.util.UUID; @TimescaleDBTsDao public interface TsKvTimescaleRepository extends CrudRepository { @@ -35,31 +36,21 @@ public interface TsKvTimescaleRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") List findAllWithLimit( - @Param("tenantId") String tenantId, - @Param("entityId") String entityId, - @Param("entityKey") String key, + @Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("entityKey") int key, @Param("startTs") long startTs, @Param("endTs") long endTs, Pageable pageable); - @Query(value = "SELECT tskv.tenant_id as tenant_id, tskv.entity_id as entity_id, tskv.key as key, last(tskv.ts,tskv.ts) as ts," + - " last(tskv.bool_v, tskv.ts) as bool_v, last(tskv.str_v, tskv.ts) as str_v," + - " last(tskv.long_v, tskv.ts) as long_v, last(tskv.dbl_v, tskv.ts) as dbl_v" + - " FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS varchar) " + - "AND tskv.entity_id = cast(:entityId AS varchar) " + - "GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key", nativeQuery = true) - List findAllLatestValues( - @Param("tenantId") String tenantId, - @Param("entityId") String entityId); - @Transactional @Modifying @Query("DELETE FROM TimescaleTsKvEntity tskv WHERE tskv.tenantId = :tenantId " + "AND tskv.entityId = :entityId " + "AND tskv.key = :entityKey " + "AND tskv.ts > :startTs AND tskv.ts <= :endTs") - void delete(@Param("tenantId") String tenantId, - @Param("entityId") String entityId, - @Param("entityKey") String key, + void delete(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("entityKey") int key, @Param("startTs") long startTs, @Param("endTs") long endTs); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlLatestInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlLatestInsertRepository.java deleted file mode 100644 index b8b4766e70..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlLatestInsertRepository.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts.ts; - -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository; -import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; - -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.List; - -@SqlTsDao -@HsqlDao -@Repository -@Transactional -public class HsqlLatestInsertRepository extends AbstractLatestInsertRepository { - - private static final String TS_KV_LATEST_CONSTRAINT = "(ts_kv_latest.entity_type=A.entity_type AND ts_kv_latest.entity_id=A.entity_id AND ts_kv_latest.key=A.key)"; - - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, BOOL_V, HSQL_LATEST_ON_BOOL_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, STR_V, HSQL_LATEST_ON_STR_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, LONG_V, HSQL_LATEST_ON_LONG_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_LATEST_TABLE, TS_KV_LATEST_CONSTRAINT, DBL_V, HSQL_LATEST_ON_DBL_VALUE_UPDATE_SET_NULLS); - - private static final String INSERT_OR_UPDATE = - "MERGE INTO ts_kv_latest USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " + - "T (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " + - "ON (ts_kv_latest.entity_type=T.entity_type " + - "AND ts_kv_latest.entity_id=T.entity_id " + - "AND ts_kv_latest.key=T.key) " + - "WHEN MATCHED THEN UPDATE SET ts_kv_latest.ts = T.ts, ts_kv_latest.bool_v = T.bool_v, ts_kv_latest.str_v = T.str_v, ts_kv_latest.long_v = T.long_v, ts_kv_latest.dbl_v = T.dbl_v " + - "WHEN NOT MATCHED THEN INSERT (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " + - "VALUES (T.entity_type, T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);"; - - @Override - public void saveOrUpdate(TsKvLatestEntity entity) { - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); - } - - @Override - public void saveOrUpdate(List entities) { - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - ps.setString(1, entities.get(i).getEntityType().name()); - ps.setString(2, entities.get(i).getEntityId()); - ps.setString(3, entities.get(i).getKey()); - ps.setLong(4, entities.get(i).getTs()); - - if (entities.get(i).getBooleanValue() != null) { - ps.setBoolean(5, entities.get(i).getBooleanValue()); - } else { - ps.setNull(5, Types.BOOLEAN); - } - - ps.setString(6, entities.get(i).getStrValue()); - - if (entities.get(i).getLongValue() != null) { - ps.setLong(7, entities.get(i).getLongValue()); - } else { - ps.setNull(7, Types.BIGINT); - } - - if (entities.get(i).getDoubleValue() != null) { - ps.setDouble(8, entities.get(i).getDoubleValue()); - } else { - ps.setNull(8, Types.DOUBLE); - } - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); - } - - @Override - protected void saveOrUpdateBoolean(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("bool_v", entity.getBooleanValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateString(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("str_v", entity.getStrValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateLong(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("long_v", entity.getLongValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateDouble(TsKvLatestEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("dbl_v", entity.getDoubleValue()) - .executeUpdate(); - } -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlTimeseriesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlTimeseriesInsertRepository.java deleted file mode 100644 index e909937c6b..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/HsqlTimeseriesInsertRepository.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts.ts; - -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; -import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; - -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.List; - -@SqlTsDao -@HsqlDao -@Repository -@Transactional -public class HsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepository { - - private static final String TS_KV_CONSTRAINT = "(ts_kv.entity_type=A.entity_type AND ts_kv.entity_id=A.entity_id AND ts_kv.key=A.key AND ts_kv.ts=A.ts)"; - - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, BOOL_V, HSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, STR_V, HSQL_ON_STR_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, LONG_V, HSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringHsql(TS_KV_TABLE, TS_KV_CONSTRAINT, DBL_V, HSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); - - private static final String INSERT_OR_UPDATE = - "MERGE INTO ts_kv USING(VALUES ?, ?, ?, ?, ?, ?, ?, ?) " + - "T (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " + - "ON (ts_kv.entity_type=T.entity_type " + - "AND ts_kv.entity_id=T.entity_id " + - "AND ts_kv.key=T.key " + - "AND ts_kv.ts=T.ts) " + - "WHEN MATCHED THEN UPDATE SET ts_kv.bool_v = T.bool_v, ts_kv.str_v = T.str_v, ts_kv.long_v = T.long_v, ts_kv.dbl_v = T.dbl_v " + - "WHEN NOT MATCHED THEN INSERT (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " + - "VALUES (T.entity_type, T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);"; - - @Override - public void saveOrUpdate(TsKvEntity entity) { - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); - } - - @Override - public void saveOrUpdate(List entities) { - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - ps.setString(1, entities.get(i).getEntityType().name()); - ps.setString(2, entities.get(i).getEntityId()); - ps.setString(3, entities.get(i).getKey()); - ps.setLong(4, entities.get(i).getTs()); - - if (entities.get(i).getBooleanValue() != null) { - ps.setBoolean(5, entities.get(i).getBooleanValue()); - } else { - ps.setNull(5, Types.BOOLEAN); - } - - ps.setString(6, entities.get(i).getStrValue()); - - if (entities.get(i).getLongValue() != null) { - ps.setLong(7, entities.get(i).getLongValue()); - } else { - ps.setNull(7, Types.BIGINT); - } - - if (entities.get(i).getDoubleValue() != null) { - ps.setDouble(8, entities.get(i).getDoubleValue()); - } else { - ps.setNull(8, Types.DOUBLE); - } - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); - } - - @Override - protected void saveOrUpdateBoolean(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("bool_v", entity.getBooleanValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateString(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("str_v", entity.getStrValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateLong(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("long_v", entity.getLongValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateDouble(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("dbl_v", entity.getDoubleValue()) - .executeUpdate(); - } -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/JpaTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/JpaTimeseriesDao.java deleted file mode 100644 index dfee164246..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/JpaTimeseriesDao.java +++ /dev/null @@ -1,436 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts.ts; - -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Sort; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.UUIDConverter; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestCompositeKey; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvLatestEntity; -import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; -import org.thingsboard.server.dao.sqlts.AbstractLatestInsertRepository; -import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; -import org.thingsboard.server.dao.timeseries.TimeseriesDao; -import org.thingsboard.server.dao.util.SqlTsDao; - -import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID; - - -@Component -@Slf4j -@SqlTsDao -public class JpaTimeseriesDao extends AbstractSqlTimeseriesDao implements TimeseriesDao { - - @Autowired - private TsKvRepository tsKvRepository; - - @Autowired - private TsKvLatestRepository tsKvLatestRepository; - - @Autowired - private AbstractTimeseriesInsertRepository insertRepository; - - @Autowired - private AbstractLatestInsertRepository insertLatestRepository; - - @Autowired - ScheduledLogExecutorComponent logExecutor; - - @Value("${sql.ts.batch_size:1000}") - private int tsBatchSize; - - @Value("${sql.ts.batch_max_delay:100}") - private long tsMaxDelay; - - @Value("${sql.ts.stats_print_interval_ms:1000}") - private long tsStatsPrintIntervalMs; - - @Value("${sql.ts_latest.batch_size:1000}") - private int tsLatestBatchSize; - - @Value("${sql.ts_latest.batch_max_delay:100}") - private long tsLatestMaxDelay; - - @Value("${sql.ts_latest.stats_print_interval_ms:1000}") - private long tsLatestStatsPrintIntervalMs; - - private TbSqlBlockingQueue tsQueue; - private TbSqlBlockingQueue tsLatestQueue; - - - @PostConstruct - private void init() { - TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() - .logName("TS") - .batchSize(tsBatchSize) - .maxDelay(tsMaxDelay) - .statsPrintIntervalMs(tsStatsPrintIntervalMs) - .build(); - tsQueue = new TbSqlBlockingQueue<>(tsParams); - tsQueue.init(logExecutor, v -> insertRepository.saveOrUpdate(v)); - - TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() - .logName("TS Latest") - .batchSize(tsLatestBatchSize) - .maxDelay(tsLatestMaxDelay) - .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) - .build(); - tsLatestQueue = new TbSqlBlockingQueue<>(tsLatestParams); - tsLatestQueue.init(logExecutor, v -> insertLatestRepository.saveOrUpdate(v)); - } - - @PreDestroy - private void destroy() { - if (tsQueue != null) { - tsQueue.destroy(); - } - - if (tsLatestQueue != null) { - tsLatestQueue.destroy(); - } - } - - @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { - return processFindAllAsync(tenantId, entityId, queries); - } - - protected ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - if (query.getAggregation() == Aggregation.NONE) { - return findAllAsyncWithLimit(entityId, query); - } else { - long stepTs = query.getStartTs(); - List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + query.getInterval(); - long ts = startTs + (endTs - startTs) / 2; - futures.add(findAndAggregateAsync(tenantId, entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); - stepTs = endTs; - } - return getTskvEntriesFuture(Futures.allAsList(futures)); - } - } - - private ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { - List> entitiesFutures = new ArrayList<>(); - String entityIdStr = fromTimeUUID(entityId.getId()); - switchAgregation(entityId, key, startTs, endTs, aggregation, entitiesFutures, entityIdStr); - - SettableFuture listenableFuture = SettableFuture.create(); - - CompletableFuture> entities = - CompletableFuture.allOf(entitiesFutures.toArray(new CompletableFuture[entitiesFutures.size()])) - .thenApply(v -> entitiesFutures.stream() - .map(CompletableFuture::join) - .collect(Collectors.toList())); - - entities.whenComplete((tsKvEntities, throwable) -> { - if (throwable != null) { - listenableFuture.setException(throwable); - } else { - TsKvEntity result = null; - for (TsKvEntity entity : tsKvEntities) { - if (entity.isNotEmpty()) { - result = entity; - break; - } - } - listenableFuture.set(result); - } - }); - return Futures.transform(listenableFuture, entity -> { - if (entity != null && entity.isNotEmpty()) { - entity.setEntityId(entityIdStr); - entity.setEntityType(entityId.getEntityType()); - entity.setKey(key); - entity.setTs(ts); - return Optional.of(DaoUtil.getData(entity)); - } else { - return Optional.empty(); - } - }); - } - - private void switchAgregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List> entitiesFutures, String entityIdStr) { - switch (aggregation) { - case AVG: - findAvg(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); - break; - case MAX: - findMax(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); - break; - case MIN: - findMin(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); - break; - case SUM: - findSum(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); - break; - case COUNT: - findCount(entityId, key, startTs, endTs, entitiesFutures, entityIdStr); - break; - default: - throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); - } - } - - private void findCount(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures, String entityIdStr) { - entitiesFutures.add(tsKvRepository.findCount( - entityIdStr, - entityId.getEntityType(), - key, - startTs, - endTs)); - } - - private void findSum(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures, String entityIdStr) { - entitiesFutures.add(tsKvRepository.findSum( - entityIdStr, - entityId.getEntityType(), - key, - startTs, - endTs)); - } - - private void findMin(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures, String entityIdStr) { - entitiesFutures.add(tsKvRepository.findStringMin( - entityIdStr, - entityId.getEntityType(), - key, - startTs, - endTs)); - entitiesFutures.add(tsKvRepository.findNumericMin( - entityIdStr, - entityId.getEntityType(), - key, - startTs, - endTs)); - } - - private void findMax(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures, String entityIdStr) { - entitiesFutures.add(tsKvRepository.findStringMax( - entityIdStr, - entityId.getEntityType(), - key, - startTs, - endTs)); - entitiesFutures.add(tsKvRepository.findNumericMax( - entityIdStr, - entityId.getEntityType(), - key, - startTs, - endTs)); - } - - private void findAvg(EntityId entityId, String key, long startTs, long endTs, List> entitiesFutures, String entityIdStr) { - entitiesFutures.add(tsKvRepository.findAvg( - entityIdStr, - entityId.getEntityType(), - key, - startTs, - endTs)); - } - - private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { - return Futures.immediateFuture( - DaoUtil.convertDataList( - tsKvRepository.findAllWithLimit( - fromTimeUUID(entityId.getId()), - entityId.getEntityType(), - query.getKey(), - query.getStartTs(), - query.getEndTs(), - new PageRequest(0, query.getLimit(), - new Sort(Sort.Direction.fromString( - query.getOrderBy()), "ts"))))); - } - - @Override - public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - TsKvLatestCompositeKey compositeKey = - new TsKvLatestCompositeKey( - entityId.getEntityType(), - fromTimeUUID(entityId.getId()), - key); - Optional entry = tsKvLatestRepository.findById(compositeKey); - TsKvEntry result; - if (entry.isPresent()) { - result = DaoUtil.getData(entry.get()); - } else { - result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); - } - return Futures.immediateFuture(result); - } - - @Override - public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - return Futures.immediateFuture( - DaoUtil.convertDataList(Lists.newArrayList( - tsKvLatestRepository.findAllByEntityTypeAndEntityId( - entityId.getEntityType(), - UUIDConverter.fromTimeUUID(entityId.getId()))))); - } - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - TsKvEntity entity = new TsKvEntity(); - entity.setEntityType(entityId.getEntityType()); - entity.setEntityId(fromTimeUUID(entityId.getId())); - entity.setTs(tsKvEntry.getTs()); - entity.setKey(tsKvEntry.getKey()); - entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); - entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); - entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); - entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - log.trace("Saving entity: {}", entity); - return tsQueue.add(entity); - } - - @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - return Futures.immediateFuture(null); - } - - @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); - latestEntity.setEntityType(entityId.getEntityType()); - latestEntity.setEntityId(fromTimeUUID(entityId.getId())); - latestEntity.setTs(tsKvEntry.getTs()); - latestEntity.setKey(tsKvEntry.getKey()); - latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); - latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); - latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); - latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - return tsLatestQueue.add(latestEntity); - } - - @Override - public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return service.submit(() -> { - tsKvRepository.delete( - fromTimeUUID(entityId.getId()), - entityId.getEntityType(), - query.getKey(), - query.getStartTs(), - query.getEndTs()); - return null; - }); - } - - @Override - public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture latestFuture = findLatest(tenantId, entityId, query.getKey()); - - ListenableFuture booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { - long ts = tsKvEntry.getTs(); - return ts > query.getStartTs() && ts <= query.getEndTs(); - }, service); - - ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); - latestEntity.setEntityType(entityId.getEntityType()); - latestEntity.setEntityId(fromTimeUUID(entityId.getId())); - latestEntity.setKey(query.getKey()); - return service.submit(() -> { - tsKvLatestRepository.delete(latestEntity); - return null; - }); - } - return Futures.immediateFuture(null); - }, service); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - Futures.addCallback(removedLatestFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - if (query.getRewriteLatestIfDeleted()) { - ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - return getNewLatestEntryFuture(tenantId, entityId, query); - } - return Futures.immediateFuture(null); - }, service); - - try { - resultFuture.set(savedLatestFuture.get()); - } catch (InterruptedException | ExecutionException e) { - log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); - } - } else { - resultFuture.set(null); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to process remove of the latest value", entityId, t); - } - }); - return resultFuture; - } - - private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); - return Futures.transformAsync(future, entryList -> { - if (entryList.size() == 1) { - return saveLatest(tenantId, entityId, entryList.get(0)); - } else { - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, service); - } - - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return service.submit(() -> null); - } -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java deleted file mode 100644 index d35f17b7c1..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sqlts.ts; - -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.sqlts.AbstractTimeseriesInsertRepository; -import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; - -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.List; - -@SqlTsDao -@PsqlDao -@Repository -@Transactional -public class PsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepository { - - private static final String TS_KV_CONSTRAINT = "(entity_type, entity_id, key, ts)"; - - private static final String INSERT_OR_UPDATE_BOOL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, BOOL_V, PSQL_ON_BOOL_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_STR_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, STR_V, PSQL_ON_STR_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_LONG_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, LONG_V, PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS); - private static final String INSERT_OR_UPDATE_DBL_STATEMENT = getInsertOrUpdateStringPsql(TS_KV_TABLE, TS_KV_CONSTRAINT, DBL_V, PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS); - - private static final String INSERT_OR_UPDATE = - "INSERT INTO ts_kv (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?) " + - "ON CONFLICT (entity_type, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?;"; - - @Override - public void saveOrUpdate(TsKvEntity entity) { - processSaveOrUpdate(entity, INSERT_OR_UPDATE_BOOL_STATEMENT, INSERT_OR_UPDATE_STR_STATEMENT, INSERT_OR_UPDATE_LONG_STATEMENT, INSERT_OR_UPDATE_DBL_STATEMENT); - } - - @Override - protected void saveOrUpdateBoolean(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("bool_v", entity.getBooleanValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateString(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("str_v", replaceNullChars(entity.getStrValue())) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateLong(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("long_v", entity.getLongValue()) - .executeUpdate(); - } - - @Override - protected void saveOrUpdateDouble(TsKvEntity entity, String query) { - entityManager.createNativeQuery(query) - .setParameter("entity_type", entity.getEntityType().name()) - .setParameter("entity_id", entity.getEntityId()) - .setParameter("key", entity.getKey()) - .setParameter("ts", entity.getTs()) - .setParameter("dbl_v", entity.getDoubleValue()) - .executeUpdate(); - } - - @Override - public void saveOrUpdate(List entities) { - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - TsKvEntity tsKvEntity = entities.get(i); - ps.setString(1, tsKvEntity.getEntityType().name()); - ps.setString(2, tsKvEntity.getEntityId()); - ps.setString(3, tsKvEntity.getKey()); - ps.setLong(4, tsKvEntity.getTs()); - - if (tsKvEntity.getBooleanValue() != null) { - ps.setBoolean(5, tsKvEntity.getBooleanValue()); - ps.setBoolean(9, tsKvEntity.getBooleanValue()); - } else { - ps.setNull(5, Types.BOOLEAN); - ps.setNull(9, Types.BOOLEAN); - } - - ps.setString(6, replaceNullChars(tsKvEntity.getStrValue())); - ps.setString(10, replaceNullChars(tsKvEntity.getStrValue())); - - - if (tsKvEntity.getLongValue() != null) { - ps.setLong(7, tsKvEntity.getLongValue()); - ps.setLong(11, tsKvEntity.getLongValue()); - } else { - ps.setNull(7, Types.BIGINT); - ps.setNull(11, Types.BIGINT); - } - - if (tsKvEntity.getDoubleValue() != null) { - ps.setDouble(8, tsKvEntity.getDoubleValue()); - ps.setDouble(12, tsKvEntity.getDoubleValue()); - } else { - ps.setNull(8, Types.DOUBLE); - ps.setNull(12, Types.DOUBLE); - } - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); - } -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 5dc2e7aef2..d0ebb49572 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -97,7 +97,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem @Value("${cassandra.query.set_null_values_enabled}") private boolean setNullValuesEnabled; - private TsPartitionDate tsFormat; + private NoSqlTsPartitionDate tsFormat; private PreparedStatement partitionInsertStmt; private PreparedStatement partitionInsertTtlStmt; @@ -120,7 +120,7 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem super.startExecutor(); if (!isInstall()) { getFetchStmt(Aggregation.NONE, DESC_ORDER); - Optional partition = TsPartitionDate.parse(partitioning); + Optional partition = NoSqlTsPartitionDate.parse(partitioning); if (partition.isPresent()) { tsFormat = partition.get(); } else { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsPartitionDate.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDate.java similarity index 87% rename from dao/src/main/java/org/thingsboard/server/dao/timeseries/TsPartitionDate.java rename to dao/src/main/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDate.java index 351a94d3cf..0ce26baae6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsPartitionDate.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/NoSqlTsPartitionDate.java @@ -21,7 +21,7 @@ import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalUnit; import java.util.Optional; -public enum TsPartitionDate { +public enum NoSqlTsPartitionDate { MINUTES("yyyy-MM-dd-HH-mm", ChronoUnit.MINUTES), HOURS("yyyy-MM-dd-HH", ChronoUnit.HOURS), DAYS("yyyy-MM-dd", ChronoUnit.DAYS), MONTHS("yyyy-MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS),INDEFINITE("",ChronoUnit.FOREVER); @@ -29,7 +29,7 @@ public enum TsPartitionDate { private final transient TemporalUnit truncateUnit; public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0,0, ZoneOffset.UTC); - TsPartitionDate(String pattern, TemporalUnit truncateUnit) { + NoSqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) { this.pattern = pattern; this.truncateUnit = truncateUnit; } @@ -56,10 +56,10 @@ public enum TsPartitionDate { } } - public static Optional parse(String name) { - TsPartitionDate partition = null; + public static Optional parse(String name) { + NoSqlTsPartitionDate partition = null; if (name != null) { - for (TsPartitionDate partitionDate : TsPartitionDate.values()) { + for (NoSqlTsPartitionDate partitionDate : NoSqlTsPartitionDate.values()) { if (partitionDate.name().equalsIgnoreCase(name)) { partition = partitionDate; break; diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java new file mode 100644 index 0000000000..5600ec6785 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/PsqlPartition.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.Data; + +import java.text.SimpleDateFormat; +import java.util.Date; + +@Data +public class PsqlPartition { + + private static final String TABLE_REGEX = "ts_kv_"; + + private long start; + private long end; + private String partitionDate; + private String query; + + public PsqlPartition(long start, long end, String pattern) { + this.start = start; + this.end = end; + this.partitionDate = new SimpleDateFormat(pattern).format(new Date(start)); + this.query = createStatement(start, end, partitionDate); + } + + private String createStatement(long start, long end, String partitionDate) { + return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (" + start + ") TO (" + end + ")"; + } +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java new file mode 100644 index 0000000000..202a6d9305 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlTsPartitionDate.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Optional; + +public enum SqlTsPartitionDate { + + MINUTES("yyyy_MM_dd_HH_mm", ChronoUnit.MINUTES), HOURS("yyyy_MM_dd_HH", ChronoUnit.HOURS), DAYS("yyyy_MM_dd", ChronoUnit.DAYS), MONTHS("yyyy_MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS), INDEFINITE("indefinite", ChronoUnit.FOREVER); + + private final String pattern; + private final transient TemporalUnit truncateUnit; + public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC); + + SqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) { + this.pattern = pattern; + this.truncateUnit = truncateUnit; + } + + public String getPattern() { + return pattern; + } + + public TemporalUnit getTruncateUnit() { + return truncateUnit; + } + + public LocalDateTime trancateTo(LocalDateTime time) { + switch (this) { + case MINUTES: + return time.truncatedTo(ChronoUnit.MINUTES); + case HOURS: + return time.truncatedTo(ChronoUnit.HOURS); + case DAYS: + return time.truncatedTo(ChronoUnit.DAYS); + case MONTHS: + return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1); + case YEARS: + return time.truncatedTo(ChronoUnit.DAYS).withDayOfYear(1); + case INDEFINITE: + return EPOCH_START; + default: + throw new RuntimeException("Failed to parse partitioning property!"); + } + } + + public LocalDateTime plusTo(LocalDateTime time) { + switch (this) { + case MINUTES: + return time.plusMinutes(1); + case HOURS: + return time.plusHours(1); + case DAYS: + return time.plusDays(1); + case MONTHS: + return time.plusMonths(1); + case YEARS: + return time.plusYears(1); + default: + throw new RuntimeException("Failed to parse partitioning property!"); + } + } + + public static Optional parse(String name) { + SqlTsPartitionDate partition = null; + if (name != null) { + for (SqlTsPartitionDate partitionDate : SqlTsPartitionDate.values()) { + if (partitionDate.name().equalsIgnoreCase(name)) { + partition = partitionDate; + break; + } + } + } + return Optional.of(partition); + } +} \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 0407ba78af..bcdc436608 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -17,7 +17,25 @@ CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; CREATE TABLE IF NOT EXISTS tenant_ts_kv ( - tenant_id varchar(31) NOT NULL, + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + CONSTRAINT ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts) +); + +CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) +); + +CREATE TABLE IF NOT EXISTS ts_kv_latest ( + entity_type varchar(255) NOT NULL, entity_id varchar(31) NOT NULL, key varchar(255) NOT NULL, ts bigint NOT NULL, @@ -25,7 +43,7 @@ CREATE TABLE IF NOT EXISTS tenant_ts_kv ( str_v varchar(10000000), long_v bigint, dbl_v double precision, - CONSTRAINT ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts) + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_type, entity_id, key) ); SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => 86400000, if_not_exists => true); \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-ts.sql b/dao/src/main/resources/sql/schema-ts-hsql.sql similarity index 100% rename from dao/src/main/resources/sql/schema-ts.sql rename to dao/src/main/resources/sql/schema-ts-hsql.sql diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql new file mode 100644 index 0000000000..bd9e3a693d --- /dev/null +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -0,0 +1,43 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS ts_kv ( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) +); + +CREATE TABLE IF NOT EXISTS ts_kv_latest ( + entity_type varchar(255) NOT NULL, + entity_id varchar(31) NOT NULL, + key varchar(255) NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_type, entity_id, key) +); \ No newline at end of file diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java index f5916c87f9..5ceee6c55d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java @@ -30,7 +30,7 @@ import org.springframework.test.context.support.DirtiesContextTestExecutionListe * Created by Valerii Sosliuk on 4/22/2017. */ @RunWith(SpringRunner.class) -@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, JpaDbunitTestConfig.class}) +@ContextConfiguration(classes = {JpaDaoConfig.class, HsqlTsDaoConfig.class, JpaDbunitTestConfig.class}) @TestPropertySource("classpath:sql-test.properties") @TestExecutionListeners({ DependencyInjectionTestExecutionListener.class, diff --git a/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java index ef8602d3fa..f4d11b328e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java @@ -30,9 +30,23 @@ public class JpaDaoTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"), + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), "sql/drop-all-tables.sql", "sql-test.properties" ); +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/system-data.sql"), +// "sql/drop-all-tables.sql", +// "sql-test.properties" +// ); + +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/system-data.sql"), +// "sql/timescale/drop-all-tables.sql", +// "sql-test.properties" +// ); + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java index fb36e08290..caddbabc35 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java @@ -30,9 +30,23 @@ public class SqlDaoServiceTestSuite { @ClassRule public static CustomSqlUnit sqlUnit = new CustomSqlUnit( - Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), + Arrays.asList("sql/schema-ts-hsql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), "sql/drop-all-tables.sql", "sql-test.properties" ); +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), +// "sql/drop-all-tables.sql", +// "sql-test.properties" +// ); + +// @ClassRule +// public static CustomSqlUnit sqlUnit = new CustomSqlUnit( +// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"), +// "sql/timescale/drop-all-tables.sql", +// "sql-test.properties" +// ); + } diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index a2fd6bb17b..765e0da3d6 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -2,7 +2,8 @@ database.ts.type=sql database.entities.type=sql sql.ts_inserts_executor_type=fixed -sql.ts_inserts_fixed_thread_pool_size=10 +sql.ts_inserts_fixed_thread_pool_size=200 +sql.ts_key_value_partitioning=MONTHS spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.show-sql=false @@ -13,4 +14,23 @@ spring.datasource.username=sa spring.datasource.password= spring.datasource.url=jdbc:hsqldb:file:/tmp/testDb;sql.enforce_size=false spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver -spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file +spring.datasource.hikari.maximumPoolSize = 50 + +#database.ts.type=timescale +#database.ts.type=sql +#database.entities.type=sql +# +#sql.ts_inserts_executor_type=fixed +#sql.ts_inserts_fixed_thread_pool_size=200 +#sql.ts_key_value_partitioning=MONTHS +# +#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true +#spring.jpa.show-sql=false +#spring.jpa.hibernate.ddl-auto=none +#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect +# +#spring.datasource.username=postgres +#spring.datasource.password=postgres +#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest +#spring.datasource.driverClassName=org.postgresql.Driver +#spring.datasource.hikari.maximumPoolSize = 50 \ No newline at end of file diff --git a/dao/src/test/resources/sql/timescale/drop-all-tables.sql b/dao/src/test/resources/sql/timescale/drop-all-tables.sql index e50d307c8e..08d018dc1b 100644 --- a/dao/src/test/resources/sql/timescale/drop-all-tables.sql +++ b/dao/src/test/resources/sql/timescale/drop-all-tables.sql @@ -13,6 +13,7 @@ DROP TABLE IF EXISTS relation; DROP TABLE IF EXISTS tb_user; DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS tenant_ts_kv; +DROP TABLE IF EXISTS ts_kv_latest; DROP TABLE IF EXISTS user_credentials; DROP TABLE IF EXISTS widget_type; DROP TABLE IF EXISTS widgets_bundle; diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index ce420481b7..ae615ef636 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -19,7 +19,7 @@ version: '2.2' services: postgres: restart: always - image: "postgres:9.6" + image: "postgres:10" ports: - "5432" environment: diff --git a/k8s/postgres.yml b/k8s/postgres.yml index 9ee1d8a72a..56679ff880 100644 --- a/k8s/postgres.yml +++ b/k8s/postgres.yml @@ -51,7 +51,7 @@ spec: containers: - name: postgres imagePullPolicy: Always - image: postgres:9.6 + image: postgres:10 ports: - containerPort: 5432 name: postgres diff --git a/msa/tb/docker-postgres/start-db.sh b/msa/tb/docker-postgres/start-db.sh index b0622a63df..e7b873fe83 100644 --- a/msa/tb/docker-postgres/start-db.sh +++ b/msa/tb/docker-postgres/start-db.sh @@ -20,10 +20,10 @@ firstlaunch=${DATA_FOLDER}/.firstlaunch if [ ! -d ${PGDATA} ]; then mkdir -p ${PGDATA} chown -R postgres:postgres ${PGDATA} - su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl initdb -U postgres' + su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl initdb -U postgres' fi -su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl -l /var/log/postgres/postgres.log -w start' +su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl -l /var/log/postgres/postgres.log -w start' if [ ! -f ${firstlaunch} ]; then su postgres -c 'psql -U postgres -d postgres -c "CREATE DATABASE thingsboard"' diff --git a/msa/tb/docker-postgres/stop-db.sh b/msa/tb/docker-postgres/stop-db.sh index 5f400cafff..fc5cb1784c 100644 --- a/msa/tb/docker-postgres/stop-db.sh +++ b/msa/tb/docker-postgres/stop-db.sh @@ -15,4 +15,4 @@ # limitations under the License. # -su postgres -c '/usr/lib/postgresql/9.6/bin/pg_ctl stop' +su postgres -c '/usr/lib/postgresql/10/bin/pg_ctl stop'