tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge with develop/2.5.1

pull/2865/head
Igor Kulikov 6 years ago
parent
2d7d83f013 bd40edfd14
commit
65fb9a91ec
9 changed files with 370 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 248
      application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql
  2. 44
      application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql
  3. 11
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  4. 4
      application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java
  5. 101
      application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
  6. 71
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
  7. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java
  8. 2
      dao/src/main/resources/sql/schema-timescale.sql
  9. 2
      dao/src/main/resources/sql/schema-ts-psql.sql

248
application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql
View File

@ -16,50 +16,85 @@
-- call create_partition_ts_kv_table();
CREATE OR REPLACE PROCEDURE create_partition_ts_kv_table() LANGUAGE plpgsql AS $$
CREATE OR REPLACE PROCEDURE create_partition_ts_kv_table()
LANGUAGE plpgsql AS
$$
BEGIN
ALTER TABLE ts_kv
RENAME TO ts_kv_old;
ALTER TABLE ts_kv_old
RENAME CONSTRAINT ts_kv_pkey TO ts_kv_pkey_old;
CREATE TABLE IF NOT EXISTS ts_kv
(
LIKE ts_kv_old
)
PARTITION BY RANGE (ts);
ALTER TABLE ts_kv
DROP COLUMN entity_type;
ALTER TABLE ts_kv
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
ALTER TABLE ts_kv
ALTER COLUMN key TYPE integer USING key::integer;
ALTER TABLE ts_kv
ADD CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts);
ALTER TABLE ts_kv
DROP CONSTRAINT IF EXISTS ts_kv_unq_key;
ALTER TABLE ts_kv
DROP CONSTRAINT IF EXISTS ts_kv_pkey;
ALTER TABLE ts_kv
ADD CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_type, entity_id, key, ts);
ALTER TABLE ts_kv
RENAME TO ts_kv_old;
ALTER TABLE ts_kv_old
RENAME CONSTRAINT ts_kv_pkey TO ts_kv_pkey_old;
CREATE TABLE IF NOT EXISTS ts_kv
(
LIKE ts_kv_old
)
PARTITION BY RANGE (ts);
ALTER TABLE ts_kv
DROP COLUMN entity_type;
ALTER TABLE ts_kv
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
ALTER TABLE ts_kv
ALTER COLUMN key TYPE integer USING key::integer;
ALTER TABLE ts_kv
ADD CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts);
CREATE TABLE IF NOT EXISTS ts_kv_indefinite PARTITION OF ts_kv DEFAULT;
END;
$$;
-- call create_new_ts_kv_latest_table();
CREATE OR REPLACE PROCEDURE create_new_ts_kv_latest_table() LANGUAGE plpgsql AS $$
CREATE OR REPLACE PROCEDURE create_new_ts_kv_latest_table()
LANGUAGE plpgsql AS
$$
BEGIN
ALTER TABLE ts_kv_latest
RENAME TO ts_kv_latest_old;
ALTER TABLE ts_kv_latest_old
RENAME CONSTRAINT ts_kv_latest_pkey TO ts_kv_latest_pkey_old;
CREATE TABLE IF NOT EXISTS ts_kv_latest
(
LIKE ts_kv_latest_old
);
ALTER TABLE ts_kv_latest
DROP COLUMN entity_type;
ALTER TABLE ts_kv_latest
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
ALTER TABLE ts_kv_latest
ALTER COLUMN key TYPE integer USING key::integer;
ALTER TABLE ts_kv_latest
ADD CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key);
IF NOT EXISTS(SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'ts_kv_latest_old') THEN
ALTER TABLE ts_kv_latest
DROP CONSTRAINT IF EXISTS ts_kv_latest_unq_key;
ALTER TABLE ts_kv_latest
DROP CONSTRAINT IF EXISTS ts_kv_latest_pkey;
ALTER TABLE ts_kv_latest
ADD CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_type, entity_id, key);
ALTER TABLE ts_kv_latest
RENAME TO ts_kv_latest_old;
ALTER TABLE ts_kv_latest_old
RENAME CONSTRAINT ts_kv_latest_pkey TO ts_kv_latest_pkey_old;
CREATE TABLE IF NOT EXISTS ts_kv_latest
(
LIKE ts_kv_latest_old
);
ALTER TABLE ts_kv_latest
DROP COLUMN entity_type;
ALTER TABLE ts_kv_latest
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
ALTER TABLE ts_kv_latest
ALTER COLUMN key TYPE integer USING key::integer;
ALTER TABLE ts_kv_latest
ADD CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key);
ELSE
RAISE NOTICE 'ts_kv_latest_old table already exists!';
IF NOT EXISTS(SELECT FROM pg_tables WHERE schemaname = 'public' AND tablename = 'ts_kv_latest') THEN
CREATE TABLE IF NOT EXISTS ts_kv_latest
(
entity_id uuid NOT NULL,
key int NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
json_v json,
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
);
END IF;
END IF;
END;
$$;
@ -93,8 +128,9 @@ BEGIN
RETURN QUERY SELECT SUBSTRING(year_date.year, 1, 4) AS partition_date,
(extract(epoch from (year_date.year)::timestamp) * 1000)::bigint AS from_ts,
(extract(epoch from (year_date.year::date + INTERVAL '1 YEAR')::timestamp) *
1000)::bigint AS to_ts
FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_01_01') AS year FROM ts_kv_old) AS year_date;
1000)::bigint AS to_ts
FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_01_01') AS year
FROM ts_kv_old) AS year_date;
ELSE
RAISE EXCEPTION 'Failed to parse partitioning property: % !', partition_type;
END CASE;
@ -103,13 +139,16 @@ $$ LANGUAGE plpgsql;
-- call create_partitions();
CREATE OR REPLACE PROCEDURE create_partitions(IN partition_type varchar) LANGUAGE plpgsql AS $$
CREATE OR REPLACE PROCEDURE create_partitions(IN partition_type varchar)
LANGUAGE plpgsql AS
$$
DECLARE
partition_date varchar;
from_ts bigint;
to_ts bigint;
partitions_cursor CURSOR FOR SELECT * FROM get_partitions_data(partition_type);
partitions_cursor CURSOR FOR SELECT *
FROM get_partitions_data(partition_type);
BEGIN
OPEN partitions_cursor;
LOOP
@ -127,21 +166,25 @@ $$;
-- call create_ts_kv_dictionary_table();
CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table() LANGUAGE plpgsql AS $$
CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table()
LANGUAGE plpgsql AS
$$
BEGIN
CREATE TABLE IF NOT EXISTS ts_kv_dictionary
(
key varchar(255) NOT NULL,
key_id serial UNIQUE,
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
);
CREATE TABLE IF NOT EXISTS ts_kv_dictionary
(
key varchar(255) NOT NULL,
key_id serial UNIQUE,
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
);
END;
$$;
-- call insert_into_dictionary();
CREATE OR REPLACE PROCEDURE insert_into_dictionary() LANGUAGE plpgsql AS $$
CREATE OR REPLACE PROCEDURE insert_into_dictionary()
LANGUAGE plpgsql AS
$$
DECLARE
insert_record RECORD;
@ -172,9 +215,11 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$
CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar)
LANGUAGE plpgsql AS
$$
BEGIN
EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
EXECUTE format('COPY (SELECT to_uuid(entity_id) AS entity_id,
ts_kv_records.key AS key,
ts_kv_records.ts AS ts,
ts_kv_records.bool_v AS bool_v,
@ -189,16 +234,19 @@ BEGIN
long_v,
dbl_v
FROM ts_kv_old
INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records) TO %L;', path_to_file);
EXECUTE format ('COPY ts_kv FROM %L', path_to_file);
INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records) TO %L;',
path_to_file);
EXECUTE format('COPY ts_kv FROM %L', path_to_file);
END
$$;
-- call insert_into_ts_kv_latest();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest(IN path_to_file varchar) LANGUAGE plpgsql AS $$
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest(IN path_to_file varchar)
LANGUAGE plpgsql AS
$$
BEGIN
EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
EXECUTE format('COPY (SELECT to_uuid(entity_id) AS entity_id,
ts_kv_latest_records.key AS key,
ts_kv_latest_records.ts AS ts,
ts_kv_latest_records.bool_v AS bool_v,
@ -213,9 +261,99 @@ BEGIN
long_v,
dbl_v
FROM ts_kv_latest_old
INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records) TO %L;', path_to_file);
EXECUTE format ('COPY ts_kv_latest FROM %L', path_to_file);
INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records) TO %L;',
path_to_file);
EXECUTE format('COPY ts_kv_latest FROM %L', path_to_file);
END;
$$;
-- call insert_into_ts_kv_cursor();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_cursor()
LANGUAGE plpgsql AS
$$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT to_uuid(entity_id) AS entity_id,
ts_kv_records.key AS key,
ts_kv_records.ts AS ts,
ts_kv_records.bool_v AS bool_v,
ts_kv_records.str_v AS str_v,
ts_kv_records.long_v AS long_v,
ts_kv_records.dbl_v AS dbl_v
FROM (SELECT entity_id AS entity_id,
key_id AS key,
ts,
bool_v,
str_v,
long_v,
dbl_v
FROM ts_kv_old
INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records;
BEGIN
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
END;
$$;
-- call insert_into_ts_kv_latest_cursor();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest_cursor()
LANGUAGE plpgsql AS
$$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT to_uuid(entity_id) AS entity_id,
ts_kv_latest_records.key AS key,
ts_kv_latest_records.ts AS ts,
ts_kv_latest_records.bool_v AS bool_v,
ts_kv_latest_records.str_v AS str_v,
ts_kv_latest_records.long_v AS long_v,
ts_kv_latest_records.dbl_v AS dbl_v
FROM (SELECT entity_id AS entity_id,
key_id AS key,
ts,
bool_v,
str_v,
long_v,
dbl_v
FROM ts_kv_latest_old
INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records;
BEGIN
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
END;
$$;

44
application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql
View File

@ -162,3 +162,47 @@ BEGIN
CLOSE insert_cursor;
END;
$$;
-- call insert_into_ts_kv_cursor();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_cursor() LANGUAGE plpgsql AS $$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT to_uuid(entity_id) AS entity_id,
new_ts_kv_records.key AS key,
new_ts_kv_records.ts AS ts,
new_ts_kv_records.bool_v AS bool_v,
new_ts_kv_records.str_v AS str_v,
new_ts_kv_records.long_v AS long_v,
new_ts_kv_records.dbl_v AS dbl_v
FROM (SELECT entity_id AS entity_id,
key_id AS key,
ts,
bool_v,
str_v,
long_v,
dbl_v
FROM tenant_ts_kv_old
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records;
BEGIN
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
END;
$$;

11
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
View File

@ -149,12 +149,13 @@ public class ThingsboardInstallService {
databaseTsUpgradeService.upgradeDatabase("2.4.3");
}
databaseEntitiesUpgradeService.upgradeDatabase("2.4.3");
log.info("Updating system data...");
systemDataLoaderService.updateSystemWidgets();
break;
case "2.5.0":
log.info("Upgrading ThingsBoard from version 2.5 to 3.0 ...");
log.info("Upgrading ThingsBoard from version 2.5.0 to 2.5.1 ...");
if (databaseTsUpgradeService != null) {
databaseTsUpgradeService.upgradeDatabase("2.5.0");
}
case "2.5.1":
log.info("Upgrading ThingsBoard from version 2.5.1 to 3.0.0 ...");
log.info("Updating system data...");
systemDataLoaderService.updateSystemWidgets();
break;

4
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java
View File

@ -37,8 +37,6 @@ public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaServic
@Override
public void createDatabaseSchema() throws Exception {
super.createDatabaseSchema();
if (partitionType.equals("INDEFINITE")) {
executeQuery("CREATE TABLE ts_kv_indefinite PARTITION OF ts_kv DEFAULT;");
}
executeQuery("CREATE TABLE IF NOT EXISTS ts_kv_indefinite PARTITION OF ts_kv DEFAULT;");
}
}

101
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
View File

@ -57,11 +57,15 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest(IN path_to_file varchar)";
private static final String INSERT_INTO_TS_KV_CURSOR = "insert_into_ts_kv_cursor()";
private static final String INSERT_INTO_TS_KV_LATEST_CURSOR = "insert_into_ts_kv_latest_cursor()";
private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE;
private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE;
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
private static final String CALL_INSERT_INTO_TS_KV_CURSOR = CALL_REGEX + INSERT_INTO_TS_KV_CURSOR;
private static final String CALL_INSERT_INTO_TS_KV_LATEST_CURSOR = CALL_REGEX + INSERT_INTO_TS_KV_LATEST_CURSOR;
private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD;
private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD;
@ -73,6 +77,8 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_CURSOR = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_CURSOR;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST_CURSOR = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST_CURSOR;
private static final String DROP_FUNCTION_GET_PARTITION_DATA = "DROP FUNCTION IF EXISTS get_partitions_data;";
@Override
@ -93,14 +99,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE);
if (!partitionType.equals("INDEFINITE")) {
executeQuery(conn, "call create_partitions('" + partitionType + "')");
} else {
executeQuery(conn, "CREATE TABLE IF NOT EXISTS ts_kv_indefinite PARTITION OF ts_kv DEFAULT;");
}
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
Path pathToTempTsKvFile;
Path pathToTempTsKvLatestFile;
Path pathToTempTsKvFile = null;
Path pathToTempTsKvLatestFile = null;
if (SystemUtils.IS_OS_WINDOWS) {
log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR);
Path pathToDir;
@ -118,39 +122,41 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
Path tsKvLatestFile = Files.createTempFile(pathToDir, "ts_kv_latest", ".sql");
pathToTempTsKvFile = tsKvFile.toAbsolutePath();
pathToTempTsKvLatestFile = tsKvLatestFile.toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
try {
copyTimeseries(conn, pathToTempTsKvFile, pathToTempTsKvLatestFile);
} catch (Exception e) {
insertTimeseries(conn);
}
} catch (IOException | SecurityException e) {
throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
log.warn("Failed to create time-series upgrade files due to: {}", e.getMessage());
insertTimeseries(conn);
}
} else {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
if (writable && readable && executable) {
try {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
pathToTempTsKvLatestFile = tempDirPath.resolve(TS_KV_LATEST_SQL).toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
}
if (pathToTempTsKvFile.toFile().exists() && pathToTempTsKvLatestFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
}
boolean deleteTsKvLatestFile = pathToTempTsKvLatestFile.toFile().delete();
if (deleteTsKvLatestFile) {
log.info("Successfully deleted the temp file for ts_kv_latest table upgrade!");
try {
if (writable && readable && executable) {
copyTimeseries(conn, pathToTempTsKvFile, pathToTempTsKvLatestFile);
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
} catch (Exception e) {
insertTimeseries(conn);
}
} catch (IOException | SecurityException e) {
log.warn("Failed to create time-series upgrade files due to: {}", e.getMessage());
insertTimeseries(conn);
}
}
removeUpgradeFiles(pathToTempTsKvFile, pathToTempTsKvLatestFile);
executeQuery(conn, DROP_TABLE_TS_KV_OLD);
executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD);
@ -161,6 +167,8 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV);
executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_CURSOR);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST_CURSOR);
executeQuery(conn, DROP_FUNCTION_GET_PARTITION_DATA);
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");
@ -181,11 +189,46 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
}
}
break;
case "2.5.0":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
executeQuery(conn, "CREATE TABLE IF NOT EXISTS ts_kv_indefinite PARTITION OF ts_kv DEFAULT;");
executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001");
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}
}
private void removeUpgradeFiles(Path pathToTempTsKvFile, Path pathToTempTsKvLatestFile) {
if (pathToTempTsKvFile != null && pathToTempTsKvFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
}
}
if (pathToTempTsKvLatestFile != null && pathToTempTsKvLatestFile.toFile().exists()) {
boolean deleteTsKvLatestFile = pathToTempTsKvLatestFile.toFile().delete();
if (deleteTsKvLatestFile) {
log.info("Successfully deleted the temp file for ts_kv_latest table upgrade!");
}
}
}
private void copyTimeseries(Connection conn, Path pathToTempTsKvFile, Path pathToTempTsKvLatestFile) {
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "')");
}
private void insertTimeseries(Connection conn) {
log.warn("Upgrade script failed using the copy to/from files strategy!" +
" Trying to perfrom the upgrade using Inserts strategy ...");
executeQuery(conn, CALL_INSERT_INTO_TS_KV_CURSOR);
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST_CURSOR);
}
@Override
protected void loadSql(Connection conn, String fileName) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName);

71
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
View File

@ -53,6 +53,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
private static final String INSERT_INTO_TS_KV_CURSOR = "insert_into_ts_kv_cursor()";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()";
private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE;
@ -60,6 +61,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
private static final String CALL_INSERT_INTO_TS_KV_CURSOR = CALL_REGEX + INSERT_INTO_TS_KV_CURSOR;
private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE;
@ -68,6 +70,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_CURSOR = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_CURSOR;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
@Autowired
@ -96,7 +99,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
Path pathToTempTsKvFile;
Path pathToTempTsKvFile = null;
if (SystemUtils.IS_OS_WINDOWS) {
Path pathToDir;
log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR);
@ -112,31 +115,38 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
try {
Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql");
pathToTempTsKvFile = tsKvFile.toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
pathToTempTsKvFile.toFile().deleteOnExit();
try {
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
} catch (Exception e) {
insertTimeseries(conn);
}
} catch (IOException | SecurityException e) {
throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
log.warn("Failed to create time-series upgrade files due to: {}", e.getMessage());
insertTimeseries(conn);
}
} else {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
if (writable && readable && executable) {
try {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
}
if (pathToTempTsKvFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
try {
if (writable && readable && executable) {
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
} catch (Exception e) {
insertTimeseries(conn);
}
} catch (IOException | SecurityException e) {
log.warn("Failed to create time-series upgrade files due to: {}", e.getMessage());
insertTimeseries(conn);
}
}
removeUpgradeFile(pathToTempTsKvFile);
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
@ -147,6 +157,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_CURSOR);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");
@ -161,11 +172,31 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
}
}
break;
case "2.5.0":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001");
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}
}
private void insertTimeseries(Connection conn) {
log.warn("Upgrade script failed using the copy to/from files strategy!" +
" Trying to perfrom the upgrade using Inserts strategy ...");
executeQuery(conn, CALL_INSERT_INTO_TS_KV_CURSOR);
}
private void removeUpgradeFile(Path pathToTempTsKvFile) {
if (pathToTempTsKvFile != null && pathToTempTsKvFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
}
}
}
@Override
protected void loadSql(Connection conn, String fileName) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName);

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java
View File

@ -90,7 +90,7 @@ public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa
}
private void savePartitionIfNotExist(long ts) {
if (!tsFormat.equals(SqlTsPartitionDate.INDEFINITE)) {
if (!tsFormat.equals(SqlTsPartitionDate.INDEFINITE) && ts >= 0) {
LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
LocalDateTime localDateTimeStart = tsFormat.trancateTo(time);
long partitionStartTs = toMills(localDateTimeStart);

2
dao/src/main/resources/sql/schema-timescale.sql
View File

@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000;
INSERT INTO tb_schema_settings (schema_version) VALUES (2005001) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005001;
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$

2
dao/src/main/resources/sql/schema-ts-psql.sql
View File

@ -53,7 +53,7 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005000;
INSERT INTO tb_schema_settings (schema_version) VALUES (2005001) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005001;
CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS

Write Preview
Loading…
Cancel
Save