Browse Source
* fixed the partion date extracting * fix imports * ts-keys dictionary for latest, hsqldb * removed AbstractSimpleSqlTimeseriesDao class & fix beanCreationException in ThingsboardInstallService * timescale-db upgrade added * added postgreSQL upgrade * fix logging * refactoring timeseries-dao implementationpull/2412/head
ShvaykaD
7 years ago
committed by
Andrew Shvayka
Andrew Shvayka
34 changed files with 1037 additions and 611 deletions
@ -0,0 +1,213 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- select check_version(); |
|||
|
|||
CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$ |
|||
DECLARE |
|||
current_version integer; |
|||
valid_version boolean; |
|||
BEGIN |
|||
RAISE NOTICE 'Check the current installed PostgreSQL version...'; |
|||
SELECT current_setting('server_version_num') INTO current_version; |
|||
IF current_version < 90600 THEN |
|||
valid_version := FALSE; |
|||
ELSE |
|||
valid_version := TRUE; |
|||
END IF; |
|||
IF valid_version = FALSE THEN |
|||
RAISE NOTICE 'Postgres version should be at least more than 9.6!'; |
|||
ELSE |
|||
RAISE NOTICE 'PostgreSQL version is valid!'; |
|||
RAISE NOTICE 'Schema update started...'; |
|||
END IF; |
|||
RETURN valid_version; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select create_tenant_ts_kv_table_copy(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_tenant_ts_kv_table_copy() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
ALTER TABLE tenant_ts_kv |
|||
RENAME TO tenant_ts_kv_old; |
|||
CREATE TABLE IF NOT EXISTS tenant_ts_kv |
|||
( |
|||
LIKE tenant_ts_kv_old |
|||
); |
|||
ALTER TABLE tenant_ts_kv |
|||
ALTER COLUMN tenant_id TYPE uuid USING tenant_id::uuid; |
|||
ALTER TABLE tenant_ts_kv |
|||
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; |
|||
ALTER TABLE tenant_ts_kv |
|||
ALTER COLUMN key TYPE integer USING key::integer; |
|||
ALTER TABLE tenant_ts_kv |
|||
ADD CONSTRAINT tenant_ts_kv_pkey PRIMARY KEY(tenant_id, entity_id, key, ts); |
|||
ALTER INDEX idx_tenant_ts_kv RENAME TO idx_tenant_ts_kv_old; |
|||
ALTER INDEX tenant_ts_kv_ts_idx RENAME TO tenant_ts_kv_ts_idx_old; |
|||
PERFORM create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => 86400000, if_not_exists => true); |
|||
CREATE INDEX IF NOT EXISTS idx_tenant_ts_kv ON tenant_ts_kv(tenant_id, entity_id, key, ts); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
|
|||
-- select create_ts_kv_latest_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_ts_kv_latest_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
CREATE TABLE IF NOT EXISTS ts_kv_latest |
|||
( |
|||
entity_id uuid NOT NULL, |
|||
key int NOT NULL, |
|||
ts bigint NOT NULL, |
|||
bool_v boolean, |
|||
str_v varchar(10000000), |
|||
long_v bigint, |
|||
dbl_v double precision, |
|||
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) |
|||
); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
|
|||
-- select create_ts_kv_dictionary_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
CREATE TABLE IF NOT EXISTS ts_kv_dictionary |
|||
( |
|||
key varchar(255) NOT NULL, |
|||
key_id serial UNIQUE, |
|||
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) |
|||
); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select insert_into_dictionary(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS |
|||
$$ |
|||
DECLARE |
|||
insert_record RECORD; |
|||
key_cursor CURSOR FOR SELECT DISTINCT key |
|||
FROM tenant_ts_kv_old |
|||
ORDER BY key; |
|||
BEGIN |
|||
OPEN key_cursor; |
|||
LOOP |
|||
FETCH key_cursor INTO insert_record; |
|||
EXIT WHEN NOT FOUND; |
|||
IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN |
|||
INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key); |
|||
RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key; |
|||
ELSE |
|||
RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE key_cursor; |
|||
END; |
|||
$$ language 'plpgsql'; |
|||
|
|||
-- select insert_into_tenant_ts_kv(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_tenant_ts_kv() RETURNS void AS |
|||
$$ |
|||
DECLARE |
|||
insert_size CONSTANT integer := 10000; |
|||
insert_counter integer DEFAULT 0; |
|||
insert_record RECORD; |
|||
insert_cursor CURSOR FOR SELECT CONCAT(tenant_id_first, '-', tenant_id_second, '-1', tenant_id_third, '-', tenant_id_fourth, '-', tenant_id_fifth)::uuid AS tenant_id, |
|||
CONCAT(entity_id_first, '-', entity_id_second, '-1', entity_id_third, '-', entity_id_fourth, '-', entity_id_fifth)::uuid AS entity_id, |
|||
substrings.key AS key, |
|||
substrings.ts AS ts, |
|||
substrings.bool_v AS bool_v, |
|||
substrings.str_v AS str_v, |
|||
substrings.long_v AS long_v, |
|||
substrings.dbl_v AS dbl_v |
|||
FROM (SELECT SUBSTRING(tenant_id, 8, 8) AS tenant_id_first, |
|||
SUBSTRING(tenant_id, 4, 4) AS tenant_id_second, |
|||
SUBSTRING(tenant_id, 1, 3) AS tenant_id_third, |
|||
SUBSTRING(tenant_id, 16, 4) AS tenant_id_fourth, |
|||
SUBSTRING(tenant_id, 20) AS tenant_id_fifth, |
|||
SUBSTRING(entity_id, 8, 8) AS entity_id_first, |
|||
SUBSTRING(entity_id, 4, 4) AS entity_id_second, |
|||
SUBSTRING(entity_id, 1, 3) AS entity_id_third, |
|||
SUBSTRING(entity_id, 16, 4) AS entity_id_fourth, |
|||
SUBSTRING(entity_id, 20) AS entity_id_fifth, |
|||
key_id AS key, |
|||
ts, |
|||
bool_v, |
|||
str_v, |
|||
long_v, |
|||
dbl_v |
|||
FROM tenant_ts_kv_old |
|||
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS substrings; |
|||
BEGIN |
|||
OPEN insert_cursor; |
|||
LOOP |
|||
insert_counter := insert_counter + 1; |
|||
FETCH insert_cursor INTO insert_record; |
|||
IF NOT FOUND THEN |
|||
RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter - 1; |
|||
EXIT; |
|||
END IF; |
|||
INSERT INTO tenant_ts_kv(tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) |
|||
VALUES (insert_record.tenant_id, insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, |
|||
insert_record.long_v, insert_record.dbl_v); |
|||
IF MOD(insert_counter, insert_size) = 0 THEN |
|||
RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE insert_cursor; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select insert_into_ts_kv_latest(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_ts_kv_latest() RETURNS void AS |
|||
$$ |
|||
DECLARE |
|||
insert_size CONSTANT integer := 10000; |
|||
insert_counter integer DEFAULT 0; |
|||
latest_record RECORD; |
|||
insert_record RECORD; |
|||
insert_cursor CURSOR FOR SELECT |
|||
latest.key AS key, |
|||
latest.entity_id AS entity_id, |
|||
latest.ts AS ts |
|||
FROM (SELECT DISTINCT key AS key, entity_id AS entity_id, MAX(ts) AS ts FROM tenant_ts_kv GROUP BY key, entity_id) AS latest; |
|||
BEGIN |
|||
OPEN insert_cursor; |
|||
LOOP |
|||
insert_counter := insert_counter + 1; |
|||
FETCH insert_cursor INTO latest_record; |
|||
IF NOT FOUND THEN |
|||
RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter - 1; |
|||
EXIT; |
|||
END IF; |
|||
SELECT entity_id AS entity_id, key AS key, ts AS ts, bool_v AS bool_v, str_v AS str_v, long_v AS long_v, dbl_v AS dbl_v INTO insert_record FROM tenant_ts_kv WHERE entity_id = latest_record.entity_id AND key = latest_record.key AND ts = latest_record.ts; |
|||
INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) |
|||
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, insert_record.long_v, insert_record.dbl_v); |
|||
IF MOD(insert_counter, insert_size) = 0 THEN |
|||
RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE insert_cursor; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
@ -0,0 +1,147 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.PsqlDao; |
|||
import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
|||
|
|||
import java.nio.charset.StandardCharsets; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
import java.sql.CallableStatement; |
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
import java.sql.SQLException; |
|||
import java.sql.Types; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@Slf4j |
|||
@TimescaleDBTsDao |
|||
@PsqlDao |
|||
public class SqlTimescaleDatabaseUpgradeService implements DatabaseTsUpgradeService { |
|||
|
|||
private static final String CALL_REGEX = "call "; |
|||
private static final String LOAD_FUNCTIONS_SQL = "schema_update_timescale_ts.sql"; |
|||
private static final String CHECK_VERSION = CALL_REGEX + "check_version()"; |
|||
private static final String CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + "create_ts_kv_latest_table()"; |
|||
private static final String CREATE_TENANT_TS_KV_TABLE_COPY = CALL_REGEX + "create_tenant_ts_kv_table_copy()"; |
|||
private static final String CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + "create_ts_kv_dictionary_table()"; |
|||
private static final String INSERT_INTO_DICTIONARY = CALL_REGEX + "insert_into_dictionary()"; |
|||
private static final String INSERT_INTO_TS_KV = CALL_REGEX + "insert_into_tenant_ts_kv()"; |
|||
private static final String INSERT_INTO_TS_KV_LATEST = CALL_REGEX + "insert_into_ts_kv_latest()"; |
|||
private static final String DROP_OLD_TS_KV_TABLE = "DROP TABLE tenant_ts_kv_old;"; |
|||
|
|||
@Value("${spring.datasource.url}") |
|||
private String dbUrl; |
|||
|
|||
@Value("${spring.datasource.username}") |
|||
private String dbUserName; |
|||
|
|||
@Value("${spring.datasource.password}") |
|||
private String dbPassword; |
|||
|
|||
@Autowired |
|||
private InstallScripts installScripts; |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
switch (fromVersion) { |
|||
case "2.4.3": |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
log.info("Updating timescale schema ..."); |
|||
log.info("Load upgrade functions ..."); |
|||
loadSql(conn); |
|||
boolean versionValid = checkVersion(conn); |
|||
if (!versionValid) { |
|||
log.info("PostgreSQL version should be at least more than 9.6!"); |
|||
log.info("Please upgrade your PostgreSQL and restart the script!"); |
|||
} else { |
|||
log.info("PostgreSQL version is valid!"); |
|||
log.info("Updating schema ..."); |
|||
executeFunction(conn, CREATE_TS_KV_LATEST_TABLE); |
|||
executeFunction(conn, CREATE_TENANT_TS_KV_TABLE_COPY); |
|||
executeFunction(conn, CREATE_TS_KV_DICTIONARY_TABLE); |
|||
executeFunction(conn, INSERT_INTO_DICTIONARY); |
|||
executeFunction(conn, INSERT_INTO_TS_KV); |
|||
executeFunction(conn, INSERT_INTO_TS_KV_LATEST); |
|||
executeQuery(conn, DROP_OLD_TS_KV_TABLE); |
|||
log.info("schema timeseries updated!"); |
|||
} |
|||
} |
|||
break; |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
private void loadSql(Connection conn) { |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL); |
|||
try { |
|||
loadFunctions(schemaUpdateFile, conn); |
|||
log.info("Upgrade functions successfully loaded!"); |
|||
} catch (Exception e) { |
|||
log.info("Failed to load Timescale upgrade functions due to: {}", e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
private void loadFunctions(Path sqlFile, Connection conn) throws Exception { |
|||
String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); |
|||
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
} |
|||
|
|||
private boolean checkVersion(Connection conn) { |
|||
log.info("Check the current PostgreSQL version..."); |
|||
boolean versionValid = false; |
|||
try { |
|||
CallableStatement callableStatement = conn.prepareCall("{? = " + CHECK_VERSION + " }"); |
|||
callableStatement.registerOutParameter(1, Types.BOOLEAN); |
|||
callableStatement.execute(); |
|||
versionValid = callableStatement.getBoolean(1); |
|||
callableStatement.close(); |
|||
} catch (Exception e) { |
|||
log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage()); |
|||
} |
|||
return versionValid; |
|||
} |
|||
|
|||
private void executeFunction(Connection conn, String query) { |
|||
log.info("{} ... ", query); |
|||
try { |
|||
CallableStatement callableStatement = conn.prepareCall("{" + query + "}"); |
|||
callableStatement.execute(); |
|||
callableStatement.close(); |
|||
log.info("Successfully executed: {}", query.replace(CALL_REGEX, "")); |
|||
} catch (Exception e) { |
|||
log.info("Failed to execute {} due to: {}", query, e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
private void executeQuery(Connection conn, String query) { |
|||
try { |
|||
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
Thread.sleep(5000); |
|||
} catch (InterruptedException | SQLException e) { |
|||
log.info("Failed to drop table {} due to: {}", query.replace("DROP TABLE ", ""), e.getMessage()); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
|
|||
@ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") |
|||
public @interface SqlTsAnyDao { |
|||
} |
|||
@ -0,0 +1,45 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.sqlts.latest; |
|||
|
|||
import org.springframework.stereotype.Repository; |
|||
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; |
|||
import org.thingsboard.server.dao.util.SqlTsAnyDao; |
|||
|
|||
import javax.persistence.EntityManager; |
|||
import javax.persistence.PersistenceContext; |
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
|
|||
@SqlTsAnyDao |
|||
@Repository |
|||
public class SearchTsKvLatestRepository { |
|||
|
|||
public static final String FIND_ALL_BY_ENTITY_ID = "findAllByEntityId"; |
|||
public static final String FIND_ALL_BY_ENTITY_ID_QUERY = "SELECT ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key, ts_kv_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue," + |
|||
" ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.ts AS ts FROM ts_kv_latest " + |
|||
"INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)"; |
|||
|
|||
@PersistenceContext |
|||
private EntityManager entityManager; |
|||
|
|||
public List<TsKvLatestEntity> findAllByEntityId(UUID entityId) { |
|||
return entityManager.createNamedQuery(FIND_ALL_BY_ENTITY_ID, TsKvLatestEntity.class) |
|||
.setParameter("id", entityId) |
|||
.getResultList(); |
|||
} |
|||
|
|||
} |
|||
Loading…
Reference in new issue