Igor Kulikov
7 years ago
947 changed files with 37087 additions and 16219 deletions
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,31 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
DROP INDEX IF EXISTS idx_alarm_originator_alarm_type; |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_alarm_originator_alarm_type ON alarm(originator_id, type, start_ts DESC); |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_device_customer_id ON device(tenant_id, customer_id); |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_device_customer_id_and_type ON device(tenant_id, customer_id, type); |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_device_type ON device(tenant_id, type); |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id); |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); |
|||
@ -0,0 +1,251 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- select check_version(); |
|||
|
|||
CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$ |
|||
DECLARE |
|||
current_version integer; |
|||
valid_version boolean; |
|||
BEGIN |
|||
RAISE NOTICE 'Check the current installed PostgreSQL version...'; |
|||
SELECT current_setting('server_version_num') INTO current_version; |
|||
IF current_version < 100000 THEN |
|||
valid_version := FALSE; |
|||
ELSE |
|||
valid_version := TRUE; |
|||
END IF; |
|||
IF valid_version = FALSE THEN |
|||
RAISE NOTICE 'Postgres version should be at least more than 10!'; |
|||
ELSE |
|||
RAISE NOTICE 'PostgreSQL version is valid!'; |
|||
RAISE NOTICE 'Schema update started...'; |
|||
END IF; |
|||
RETURN valid_version; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select create_partition_ts_kv_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_partition_ts_kv_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
ALTER TABLE ts_kv |
|||
RENAME TO ts_kv_old; |
|||
CREATE TABLE IF NOT EXISTS ts_kv |
|||
( |
|||
LIKE ts_kv_old |
|||
) |
|||
PARTITION BY RANGE (ts); |
|||
ALTER TABLE ts_kv |
|||
DROP COLUMN entity_type; |
|||
ALTER TABLE ts_kv |
|||
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; |
|||
ALTER TABLE ts_kv |
|||
ALTER COLUMN key TYPE integer USING key::integer; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select create_new_ts_kv_latest_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_new_ts_kv_latest_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
ALTER TABLE ts_kv_latest |
|||
RENAME TO ts_kv_latest_old; |
|||
ALTER TABLE ts_kv_latest_old |
|||
RENAME CONSTRAINT ts_kv_latest_pkey TO ts_kv_latest_pkey_old; |
|||
CREATE TABLE IF NOT EXISTS ts_kv_latest |
|||
( |
|||
LIKE ts_kv_latest_old |
|||
); |
|||
ALTER TABLE ts_kv_latest |
|||
DROP COLUMN entity_type; |
|||
ALTER TABLE ts_kv_latest |
|||
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; |
|||
ALTER TABLE ts_kv_latest |
|||
ALTER COLUMN key TYPE integer USING key::integer; |
|||
ALTER TABLE ts_kv_latest |
|||
ADD CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
|
|||
-- select create_partitions(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_partitions() RETURNS VOID AS |
|||
$$ |
|||
DECLARE |
|||
partition_date varchar; |
|||
from_ts bigint; |
|||
to_ts bigint; |
|||
key_cursor CURSOR FOR select SUBSTRING(month_date.first_date, 1, 7) AS partition_date, |
|||
extract(epoch from (month_date.first_date)::timestamp) * 1000 as from_ts, |
|||
extract(epoch from (month_date.first_date::date + INTERVAL '1 MONTH')::timestamp) * |
|||
1000 as to_ts |
|||
FROM (SELECT DISTINCT TO_CHAR(TO_TIMESTAMP(ts / 1000), 'YYYY_MM_01') AS first_date |
|||
FROM ts_kv_old) AS month_date; |
|||
BEGIN |
|||
OPEN key_cursor; |
|||
LOOP |
|||
FETCH key_cursor INTO partition_date, from_ts, to_ts; |
|||
EXIT WHEN NOT FOUND; |
|||
EXECUTE 'CREATE TABLE IF NOT EXISTS ts_kv_' || partition_date || |
|||
' PARTITION OF ts_kv(PRIMARY KEY (entity_id, key, ts)) FOR VALUES FROM (' || from_ts || |
|||
') TO (' || to_ts || ');'; |
|||
RAISE NOTICE 'A partition % has been created!',CONCAT('ts_kv_', partition_date); |
|||
END LOOP; |
|||
|
|||
CLOSE key_cursor; |
|||
END; |
|||
$$ language 'plpgsql'; |
|||
|
|||
-- select create_ts_kv_dictionary_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
CREATE TABLE IF NOT EXISTS ts_kv_dictionary |
|||
( |
|||
key varchar(255) NOT NULL, |
|||
key_id serial UNIQUE, |
|||
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) |
|||
); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select insert_into_dictionary(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS |
|||
$$ |
|||
DECLARE |
|||
insert_record RECORD; |
|||
key_cursor CURSOR FOR SELECT DISTINCT key |
|||
FROM ts_kv_old |
|||
ORDER BY key; |
|||
BEGIN |
|||
OPEN key_cursor; |
|||
LOOP |
|||
FETCH key_cursor INTO insert_record; |
|||
EXIT WHEN NOT FOUND; |
|||
IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN |
|||
INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key); |
|||
RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key; |
|||
ELSE |
|||
RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE key_cursor; |
|||
END; |
|||
$$ language 'plpgsql'; |
|||
|
|||
-- select insert_into_ts_kv(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_ts_kv() RETURNS void AS |
|||
$$ |
|||
DECLARE |
|||
insert_size CONSTANT integer := 10000; |
|||
insert_counter integer DEFAULT 0; |
|||
insert_record RECORD; |
|||
insert_cursor CURSOR FOR SELECT CONCAT(first_part_uuid, '-', second_part_uuid, '-1', third_part_uuid, '-', fourth_part_uuid, '-', fifth_part_uuid)::uuid AS entity_id, |
|||
ts_kv_records.key AS key, |
|||
ts_kv_records.ts AS ts, |
|||
ts_kv_records.bool_v AS bool_v, |
|||
ts_kv_records.str_v AS str_v, |
|||
ts_kv_records.long_v AS long_v, |
|||
ts_kv_records.dbl_v AS dbl_v |
|||
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first_part_uuid, |
|||
SUBSTRING(entity_id, 4, 4) AS second_part_uuid, |
|||
SUBSTRING(entity_id, 1, 3) AS third_part_uuid, |
|||
SUBSTRING(entity_id, 16, 4) AS fourth_part_uuid, |
|||
SUBSTRING(entity_id, 20) AS fifth_part_uuid, |
|||
key_id AS key, |
|||
ts, |
|||
bool_v, |
|||
str_v, |
|||
long_v, |
|||
dbl_v |
|||
FROM ts_kv_old |
|||
INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records; |
|||
BEGIN |
|||
OPEN insert_cursor; |
|||
LOOP |
|||
insert_counter := insert_counter + 1; |
|||
FETCH insert_cursor INTO insert_record; |
|||
IF NOT FOUND THEN |
|||
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1; |
|||
EXIT; |
|||
END IF; |
|||
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) |
|||
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, |
|||
insert_record.long_v, insert_record.dbl_v); |
|||
IF MOD(insert_counter, insert_size) = 0 THEN |
|||
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE insert_cursor; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select insert_into_ts_kv_latest(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_ts_kv_latest() RETURNS void AS |
|||
$$ |
|||
DECLARE |
|||
insert_size CONSTANT integer := 10000; |
|||
insert_counter integer DEFAULT 0; |
|||
insert_record RECORD; |
|||
insert_cursor CURSOR FOR SELECT CONCAT(first_part_uuid, '-', second_part_uuid, '-1', third_part_uuid, '-', fourth_part_uuid, '-', fifth_part_uuid)::uuid AS entity_id, |
|||
ts_kv_latest_records.key AS key, |
|||
ts_kv_latest_records.ts AS ts, |
|||
ts_kv_latest_records.bool_v AS bool_v, |
|||
ts_kv_latest_records.str_v AS str_v, |
|||
ts_kv_latest_records.long_v AS long_v, |
|||
ts_kv_latest_records.dbl_v AS dbl_v |
|||
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first_part_uuid, |
|||
SUBSTRING(entity_id, 4, 4) AS second_part_uuid, |
|||
SUBSTRING(entity_id, 1, 3) AS third_part_uuid, |
|||
SUBSTRING(entity_id, 16, 4) AS fourth_part_uuid, |
|||
SUBSTRING(entity_id, 20) AS fifth_part_uuid, |
|||
key_id AS key, |
|||
ts, |
|||
bool_v, |
|||
str_v, |
|||
long_v, |
|||
dbl_v |
|||
FROM ts_kv_latest_old |
|||
INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records; |
|||
BEGIN |
|||
OPEN insert_cursor; |
|||
LOOP |
|||
insert_counter := insert_counter + 1; |
|||
FETCH insert_cursor INTO insert_record; |
|||
IF NOT FOUND THEN |
|||
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter - 1; |
|||
EXIT; |
|||
END IF; |
|||
INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) |
|||
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, |
|||
insert_record.long_v, insert_record.dbl_v); |
|||
IF MOD(insert_counter, insert_size) = 0 THEN |
|||
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE insert_cursor; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
|
|||
@ -0,0 +1,213 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- select check_version(); |
|||
|
|||
CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$ |
|||
DECLARE |
|||
current_version integer; |
|||
valid_version boolean; |
|||
BEGIN |
|||
RAISE NOTICE 'Check the current installed PostgreSQL version...'; |
|||
SELECT current_setting('server_version_num') INTO current_version; |
|||
IF current_version < 90600 THEN |
|||
valid_version := FALSE; |
|||
ELSE |
|||
valid_version := TRUE; |
|||
END IF; |
|||
IF valid_version = FALSE THEN |
|||
RAISE NOTICE 'Postgres version should be at least more than 9.6!'; |
|||
ELSE |
|||
RAISE NOTICE 'PostgreSQL version is valid!'; |
|||
RAISE NOTICE 'Schema update started...'; |
|||
END IF; |
|||
RETURN valid_version; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select create_new_tenant_ts_kv_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_new_tenant_ts_kv_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
ALTER TABLE tenant_ts_kv |
|||
RENAME TO tenant_ts_kv_old; |
|||
CREATE TABLE IF NOT EXISTS tenant_ts_kv |
|||
( |
|||
LIKE tenant_ts_kv_old |
|||
); |
|||
ALTER TABLE tenant_ts_kv |
|||
ALTER COLUMN tenant_id TYPE uuid USING tenant_id::uuid; |
|||
ALTER TABLE tenant_ts_kv |
|||
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid; |
|||
ALTER TABLE tenant_ts_kv |
|||
ALTER COLUMN key TYPE integer USING key::integer; |
|||
ALTER TABLE tenant_ts_kv |
|||
ADD CONSTRAINT tenant_ts_kv_pkey PRIMARY KEY(tenant_id, entity_id, key, ts); |
|||
ALTER INDEX idx_tenant_ts_kv RENAME TO idx_tenant_ts_kv_old; |
|||
ALTER INDEX tenant_ts_kv_ts_idx RENAME TO tenant_ts_kv_ts_idx_old; |
|||
-- PERFORM create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => 86400000, if_not_exists => true); |
|||
CREATE INDEX IF NOT EXISTS idx_tenant_ts_kv ON tenant_ts_kv(tenant_id, entity_id, key, ts); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
|
|||
-- select create_ts_kv_latest_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_ts_kv_latest_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
CREATE TABLE IF NOT EXISTS ts_kv_latest |
|||
( |
|||
entity_id uuid NOT NULL, |
|||
key int NOT NULL, |
|||
ts bigint NOT NULL, |
|||
bool_v boolean, |
|||
str_v varchar(10000000), |
|||
long_v bigint, |
|||
dbl_v double precision, |
|||
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) |
|||
); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
|
|||
-- select create_ts_kv_dictionary_table(); |
|||
|
|||
CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$ |
|||
|
|||
BEGIN |
|||
CREATE TABLE IF NOT EXISTS ts_kv_dictionary |
|||
( |
|||
key varchar(255) NOT NULL, |
|||
key_id serial UNIQUE, |
|||
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) |
|||
); |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select insert_into_dictionary(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS |
|||
$$ |
|||
DECLARE |
|||
insert_record RECORD; |
|||
key_cursor CURSOR FOR SELECT DISTINCT key |
|||
FROM tenant_ts_kv_old |
|||
ORDER BY key; |
|||
BEGIN |
|||
OPEN key_cursor; |
|||
LOOP |
|||
FETCH key_cursor INTO insert_record; |
|||
EXIT WHEN NOT FOUND; |
|||
IF NOT EXISTS(SELECT key FROM ts_kv_dictionary WHERE key = insert_record.key) THEN |
|||
INSERT INTO ts_kv_dictionary(key) VALUES (insert_record.key); |
|||
RAISE NOTICE 'Key: % has been inserted into the dictionary!',insert_record.key; |
|||
ELSE |
|||
RAISE NOTICE 'Key: % already exists in the dictionary!',insert_record.key; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE key_cursor; |
|||
END; |
|||
$$ language 'plpgsql'; |
|||
|
|||
-- select insert_into_tenant_ts_kv(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_tenant_ts_kv() RETURNS void AS |
|||
$$ |
|||
DECLARE |
|||
insert_size CONSTANT integer := 10000; |
|||
insert_counter integer DEFAULT 0; |
|||
insert_record RECORD; |
|||
insert_cursor CURSOR FOR SELECT CONCAT(tenant_id_first_part_uuid, '-', tenant_id_second_part_uuid, '-1', tenant_id_third_part_uuid, '-', tenant_id_fourth_part_uuid, '-', tenant_id_fifth_part_uuid)::uuid AS tenant_id, |
|||
CONCAT(entity_id_first_part_uuid, '-', entity_id_second_part_uuid, '-1', entity_id_third_part_uuid, '-', entity_id_fourth_part_uuid, '-', entity_id_fifth_part_uuid)::uuid AS entity_id, |
|||
tenant_ts_kv_records.key AS key, |
|||
tenant_ts_kv_records.ts AS ts, |
|||
tenant_ts_kv_records.bool_v AS bool_v, |
|||
tenant_ts_kv_records.str_v AS str_v, |
|||
tenant_ts_kv_records.long_v AS long_v, |
|||
tenant_ts_kv_records.dbl_v AS dbl_v |
|||
FROM (SELECT SUBSTRING(tenant_id, 8, 8) AS tenant_id_first_part_uuid, |
|||
SUBSTRING(tenant_id, 4, 4) AS tenant_id_second_part_uuid, |
|||
SUBSTRING(tenant_id, 1, 3) AS tenant_id_third_part_uuid, |
|||
SUBSTRING(tenant_id, 16, 4) AS tenant_id_fourth_part_uuid, |
|||
SUBSTRING(tenant_id, 20) AS tenant_id_fifth_part_uuid, |
|||
SUBSTRING(entity_id, 8, 8) AS entity_id_first_part_uuid, |
|||
SUBSTRING(entity_id, 4, 4) AS entity_id_second_part_uuid, |
|||
SUBSTRING(entity_id, 1, 3) AS entity_id_third_part_uuid, |
|||
SUBSTRING(entity_id, 16, 4) AS entity_id_fourth_part_uuid, |
|||
SUBSTRING(entity_id, 20) AS entity_id_fifth_part_uuid, |
|||
key_id AS key, |
|||
ts, |
|||
bool_v, |
|||
str_v, |
|||
long_v, |
|||
dbl_v |
|||
FROM tenant_ts_kv_old |
|||
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS tenant_ts_kv_records; |
|||
BEGIN |
|||
OPEN insert_cursor; |
|||
LOOP |
|||
insert_counter := insert_counter + 1; |
|||
FETCH insert_cursor INTO insert_record; |
|||
IF NOT FOUND THEN |
|||
RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter - 1; |
|||
EXIT; |
|||
END IF; |
|||
INSERT INTO tenant_ts_kv(tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v) |
|||
VALUES (insert_record.tenant_id, insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, |
|||
insert_record.long_v, insert_record.dbl_v); |
|||
IF MOD(insert_counter, insert_size) = 0 THEN |
|||
RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE insert_cursor; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
|
|||
-- select insert_into_ts_kv_latest(); |
|||
|
|||
CREATE OR REPLACE FUNCTION insert_into_ts_kv_latest() RETURNS void AS |
|||
$$ |
|||
DECLARE |
|||
insert_size CONSTANT integer := 10000; |
|||
insert_counter integer DEFAULT 0; |
|||
latest_record RECORD; |
|||
insert_record RECORD; |
|||
insert_cursor CURSOR FOR SELECT |
|||
latest_records.key AS key, |
|||
latest_records.entity_id AS entity_id, |
|||
latest_records.ts AS ts |
|||
FROM (SELECT DISTINCT key AS key, entity_id AS entity_id, MAX(ts) AS ts FROM tenant_ts_kv GROUP BY key, entity_id) AS latest_records; |
|||
BEGIN |
|||
OPEN insert_cursor; |
|||
LOOP |
|||
insert_counter := insert_counter + 1; |
|||
FETCH insert_cursor INTO latest_record; |
|||
IF NOT FOUND THEN |
|||
RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter - 1; |
|||
EXIT; |
|||
END IF; |
|||
SELECT entity_id AS entity_id, key AS key, ts AS ts, bool_v AS bool_v, str_v AS str_v, long_v AS long_v, dbl_v AS dbl_v INTO insert_record FROM tenant_ts_kv WHERE entity_id = latest_record.entity_id AND key = latest_record.key AND ts = latest_record.ts; |
|||
INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) |
|||
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, insert_record.long_v, insert_record.dbl_v); |
|||
IF MOD(insert_counter, insert_size) = 0 THEN |
|||
RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter; |
|||
END IF; |
|||
END LOOP; |
|||
CLOSE insert_cursor; |
|||
END; |
|||
$$ LANGUAGE 'plpgsql'; |
|||
@ -0,0 +1,48 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Qualifier; |
|||
import org.thingsboard.server.dao.cassandra.CassandraCluster; |
|||
import org.thingsboard.server.dao.cassandra.CassandraInstallCluster; |
|||
import org.thingsboard.server.service.install.cql.CQLStatementsParser; |
|||
|
|||
import java.nio.file.Path; |
|||
import java.util.List; |
|||
|
|||
@Slf4j |
|||
public abstract class AbstractCassandraDatabaseUpgradeService { |
|||
@Autowired |
|||
protected CassandraCluster cluster; |
|||
|
|||
@Autowired |
|||
@Qualifier("CassandraInstallCluster") |
|||
private CassandraInstallCluster installCluster; |
|||
|
|||
protected void loadCql(Path cql) throws Exception { |
|||
List<String> statements = new CQLStatementsParser(cql).getStatements(); |
|||
statements.forEach(statement -> { |
|||
installCluster.getSession().execute(statement); |
|||
try { |
|||
Thread.sleep(2500); |
|||
} catch (InterruptedException e) { |
|||
} |
|||
}); |
|||
Thread.sleep(5000); |
|||
} |
|||
} |
|||
@ -0,0 +1,124 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
|
|||
import java.nio.charset.StandardCharsets; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.sql.CallableStatement; |
|||
import java.sql.Connection; |
|||
import java.sql.SQLException; |
|||
import java.sql.Types; |
|||
|
|||
@Slf4j |
|||
public abstract class AbstractSqlTsDatabaseUpgradeService { |
|||
|
|||
protected static final String CALL_REGEX = "call "; |
|||
protected static final String CHECK_VERSION = "check_version()"; |
|||
protected static final String DROP_TABLE = "DROP TABLE "; |
|||
protected static final String DROP_FUNCTION_IF_EXISTS = "DROP FUNCTION IF EXISTS "; |
|||
|
|||
private static final String CALL_CHECK_VERSION = CALL_REGEX + CHECK_VERSION; |
|||
|
|||
|
|||
private static final String FUNCTION = "function: {}"; |
|||
private static final String DROP_STATEMENT = "drop statement: {}"; |
|||
private static final String QUERY = "query: {}"; |
|||
private static final String SUCCESSFULLY_EXECUTED = "Successfully executed "; |
|||
private static final String FAILED_TO_EXECUTE = "Failed to execute "; |
|||
private static final String FAILED_DUE_TO = " due to: {}"; |
|||
|
|||
protected static final String SUCCESSFULLY_EXECUTED_FUNCTION = SUCCESSFULLY_EXECUTED + FUNCTION; |
|||
protected static final String FAILED_TO_EXECUTE_FUNCTION_DUE_TO = FAILED_TO_EXECUTE + FUNCTION + FAILED_DUE_TO; |
|||
|
|||
protected static final String SUCCESSFULLY_EXECUTED_DROP_STATEMENT = SUCCESSFULLY_EXECUTED + DROP_STATEMENT; |
|||
protected static final String FAILED_TO_EXECUTE_DROP_STATEMENT = FAILED_TO_EXECUTE + DROP_STATEMENT + FAILED_DUE_TO; |
|||
|
|||
protected static final String SUCCESSFULLY_EXECUTED_QUERY = SUCCESSFULLY_EXECUTED + QUERY; |
|||
protected static final String FAILED_TO_EXECUTE_QUERY = FAILED_TO_EXECUTE + QUERY + FAILED_DUE_TO; |
|||
|
|||
@Value("${spring.datasource.url}") |
|||
protected String dbUrl; |
|||
|
|||
@Value("${spring.datasource.username}") |
|||
protected String dbUserName; |
|||
|
|||
@Value("${spring.datasource.password}") |
|||
protected String dbPassword; |
|||
|
|||
@Autowired |
|||
protected InstallScripts installScripts; |
|||
|
|||
protected abstract void loadSql(Connection conn); |
|||
|
|||
protected void loadFunctions(Path sqlFile, Connection conn) throws Exception { |
|||
String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); |
|||
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
} |
|||
|
|||
protected boolean checkVersion(Connection conn) { |
|||
log.info("Check the current PostgreSQL version..."); |
|||
boolean versionValid = false; |
|||
try { |
|||
CallableStatement callableStatement = conn.prepareCall("{? = " + CALL_CHECK_VERSION + " }"); |
|||
callableStatement.registerOutParameter(1, Types.BOOLEAN); |
|||
callableStatement.execute(); |
|||
versionValid = callableStatement.getBoolean(1); |
|||
callableStatement.close(); |
|||
} catch (Exception e) { |
|||
log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage()); |
|||
} |
|||
return versionValid; |
|||
} |
|||
|
|||
protected void executeFunction(Connection conn, String query) { |
|||
log.info("{} ... ", query); |
|||
try { |
|||
CallableStatement callableStatement = conn.prepareCall("{" + query + "}"); |
|||
callableStatement.execute(); |
|||
callableStatement.close(); |
|||
log.info(SUCCESSFULLY_EXECUTED_FUNCTION, query.replace(CALL_REGEX, "")); |
|||
Thread.sleep(2000); |
|||
} catch (Exception e) { |
|||
log.info(FAILED_TO_EXECUTE_FUNCTION_DUE_TO, query, e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
protected void executeDropStatement(Connection conn, String query) { |
|||
try { |
|||
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
log.info(SUCCESSFULLY_EXECUTED_DROP_STATEMENT, query); |
|||
Thread.sleep(5000); |
|||
} catch (InterruptedException | SQLException e) { |
|||
log.info(FAILED_TO_EXECUTE_DROP_STATEMENT, query, e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
protected void executeQuery(Connection conn, String query) { |
|||
try { |
|||
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
log.info(SUCCESSFULLY_EXECUTED_QUERY, query); |
|||
Thread.sleep(5000); |
|||
} catch (InterruptedException | SQLException e) { |
|||
log.info(FAILED_TO_EXECUTE_QUERY, query, e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import com.datastax.driver.core.exceptions.InvalidQueryException; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.NoSqlTsDao; |
|||
|
|||
@Service |
|||
@NoSqlTsDao |
|||
@Profile("install") |
|||
@Slf4j |
|||
public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabaseUpgradeService implements DatabaseTsUpgradeService { |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
switch (fromVersion) { |
|||
case "2.4.3": |
|||
log.info("Updating schema ..."); |
|||
String updateTsKvTableStmt = "alter table ts_kv_cf add json_v text"; |
|||
String updateTsKvLatestTableStmt = "alter table ts_kv_latest_cf add json_v text"; |
|||
|
|||
try { |
|||
log.info("Updating ts ..."); |
|||
cluster.getSession().execute(updateTsKvTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Ts updated."); |
|||
log.info("Updating ts latest ..."); |
|||
cluster.getSession().execute(updateTsKvLatestTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Ts latest updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,32 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.PsqlDao; |
|||
import org.thingsboard.server.dao.util.SqlTsDao; |
|||
|
|||
@Service |
|||
@SqlTsDao |
|||
@PsqlDao |
|||
@Profile("install") |
|||
public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService |
|||
implements TsDatabaseSchemaService { |
|||
public PsqlTsDatabaseSchemaService() { |
|||
super("schema-ts-psql.sql", null); |
|||
} |
|||
} |
|||
@ -0,0 +1,125 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.PsqlDao; |
|||
import org.thingsboard.server.dao.util.SqlTsDao; |
|||
|
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@Slf4j |
|||
@SqlTsDao |
|||
@PsqlDao |
|||
public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { |
|||
|
|||
private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; |
|||
|
|||
private static final String TS_KV_OLD = "ts_kv_old;"; |
|||
private static final String TS_KV_LATEST_OLD = "ts_kv_latest_old;"; |
|||
|
|||
private static final String CREATE_PARTITION_TS_KV_TABLE = "create_partition_ts_kv_table()"; |
|||
private static final String CREATE_NEW_TS_KV_LATEST_TABLE = "create_new_ts_kv_latest_table()"; |
|||
private static final String CREATE_PARTITIONS = "create_partitions()"; |
|||
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; |
|||
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; |
|||
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; |
|||
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; |
|||
|
|||
private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE; |
|||
private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE; |
|||
private static final String CALL_CREATE_PARTITIONS = CALL_REGEX + CREATE_PARTITIONS; |
|||
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; |
|||
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; |
|||
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV; |
|||
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; |
|||
|
|||
private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD; |
|||
private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD; |
|||
|
|||
private static final String DROP_FUNCTION_CHECK_VERSION = DROP_FUNCTION_IF_EXISTS + CHECK_VERSION; |
|||
private static final String DROP_FUNCTION_CREATE_PARTITION_TS_KV_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_PARTITION_TS_KV_TABLE; |
|||
private static final String DROP_FUNCTION_CREATE_NEW_TS_KV_LATEST_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_NEW_TS_KV_LATEST_TABLE; |
|||
private static final String DROP_FUNCTION_CREATE_PARTITIONS = DROP_FUNCTION_IF_EXISTS + CREATE_PARTITIONS; |
|||
private static final String DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE; |
|||
private static final String DROP_FUNCTION_INSERT_INTO_DICTIONARY = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_DICTIONARY; |
|||
private static final String DROP_FUNCTION_INSERT_INTO_TS_KV = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV; |
|||
private static final String DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
switch (fromVersion) { |
|||
case "2.4.3": |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
log.info("Updating timeseries schema ..."); |
|||
log.info("Load upgrade functions ..."); |
|||
loadSql(conn); |
|||
boolean versionValid = checkVersion(conn); |
|||
if (!versionValid) { |
|||
log.info("PostgreSQL version should be at least more than 10!"); |
|||
log.info("Please upgrade your PostgreSQL and restart the script!"); |
|||
} else { |
|||
log.info("PostgreSQL version is valid!"); |
|||
log.info("Updating schema ..."); |
|||
executeFunction(conn, CALL_CREATE_PARTITION_TS_KV_TABLE); |
|||
executeFunction(conn, CALL_CREATE_PARTITIONS); |
|||
executeFunction(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); |
|||
executeFunction(conn, CALL_INSERT_INTO_DICTIONARY); |
|||
executeFunction(conn, CALL_INSERT_INTO_TS_KV); |
|||
executeFunction(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); |
|||
executeFunction(conn, CALL_INSERT_INTO_TS_KV_LATEST); |
|||
|
|||
executeDropStatement(conn, DROP_TABLE_TS_KV_OLD); |
|||
executeDropStatement(conn, DROP_TABLE_TS_KV_LATEST_OLD); |
|||
|
|||
executeDropStatement(conn, DROP_FUNCTION_CHECK_VERSION); |
|||
executeDropStatement(conn, DROP_FUNCTION_CREATE_PARTITION_TS_KV_TABLE); |
|||
executeDropStatement(conn, DROP_FUNCTION_CREATE_PARTITIONS); |
|||
executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE); |
|||
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_DICTIONARY); |
|||
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV); |
|||
executeDropStatement(conn, DROP_FUNCTION_CREATE_NEW_TS_KV_LATEST_TABLE); |
|||
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST); |
|||
|
|||
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN json_v json;"); |
|||
executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;"); |
|||
|
|||
log.info("schema timeseries updated!"); |
|||
} |
|||
} |
|||
break; |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
protected void loadSql(Connection conn) { |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL); |
|||
try { |
|||
loadFunctions(schemaUpdateFile, conn); |
|||
log.info("Upgrade functions successfully loaded!"); |
|||
} catch (Exception e) { |
|||
log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,68 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.PsqlDao; |
|||
import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
|||
|
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
import java.sql.SQLException; |
|||
|
|||
@Service |
|||
@TimescaleDBTsDao |
|||
@PsqlDao |
|||
@Profile("install") |
|||
@Slf4j |
|||
public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService { |
|||
|
|||
private static final String QUERY = "query: {}"; |
|||
private static final String SUCCESSFULLY_EXECUTED = "Successfully executed "; |
|||
private static final String FAILED_TO_EXECUTE = "Failed to execute "; |
|||
private static final String FAILED_DUE_TO = " due to: {}"; |
|||
|
|||
private static final String SUCCESSFULLY_EXECUTED_QUERY = SUCCESSFULLY_EXECUTED + QUERY; |
|||
private static final String FAILED_TO_EXECUTE_QUERY = FAILED_TO_EXECUTE + QUERY + FAILED_DUE_TO; |
|||
|
|||
@Value("${sql.timescale.chunk_time_interval:86400000}") |
|||
private long chunkTimeInterval; |
|||
|
|||
public TimescaleTsDatabaseSchemaService() { |
|||
super("schema-timescale.sql", "schema-timescale-idx.sql"); |
|||
} |
|||
|
|||
@Override |
|||
public void createDatabaseSchema() throws Exception { |
|||
super.createDatabaseSchema(); |
|||
executeQuery("SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);"); |
|||
} |
|||
|
|||
private void executeQuery(String query) { |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
log.info(SUCCESSFULLY_EXECUTED_QUERY, query); |
|||
Thread.sleep(5000); |
|||
} catch (InterruptedException | SQLException e) { |
|||
log.info(FAILED_TO_EXECUTE_QUERY, query, e.getMessage()); |
|||
} |
|||
} |
|||
|
|||
|
|||
} |
|||
@ -0,0 +1,128 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.PsqlDao; |
|||
import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
|||
|
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@Slf4j |
|||
@TimescaleDBTsDao |
|||
@PsqlDao |
|||
public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { |
|||
|
|||
@Value("${sql.timescale.chunk_time_interval:86400000}") |
|||
private long chunkTimeInterval; |
|||
|
|||
private static final String LOAD_FUNCTIONS_SQL = "schema_update_timescale_ts.sql"; |
|||
|
|||
private static final String TENANT_TS_KV_OLD_TABLE = "tenant_ts_kv_old;"; |
|||
|
|||
private static final String CREATE_TS_KV_LATEST_TABLE = "create_ts_kv_latest_table()"; |
|||
private static final String CREATE_NEW_TENANT_TS_KV_TABLE = "create_new_tenant_ts_kv_table()"; |
|||
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; |
|||
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; |
|||
private static final String INSERT_INTO_TENANT_TS_KV = "insert_into_tenant_ts_kv()"; |
|||
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; |
|||
|
|||
private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE; |
|||
private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TENANT_TS_KV_TABLE; |
|||
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; |
|||
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; |
|||
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TENANT_TS_KV; |
|||
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; |
|||
|
|||
private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE; |
|||
|
|||
private static final String DROP_FUNCTION_CREATE_TS_KV_LATEST_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_LATEST_TABLE; |
|||
private static final String DROP_FUNCTION_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_FUNCTION_IF_EXISTS + CREATE_NEW_TENANT_TS_KV_TABLE; |
|||
private static final String DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE; |
|||
private static final String DROP_FUNCTION_INSERT_INTO_DICTIONARY = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_DICTIONARY; |
|||
private static final String DROP_FUNCTION_INSERT_INTO_TENANT_TS_KV = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TENANT_TS_KV; |
|||
private static final String DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; |
|||
|
|||
@Autowired |
|||
private InstallScripts installScripts; |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
switch (fromVersion) { |
|||
case "2.4.3": |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
log.info("Updating timescale schema ..."); |
|||
log.info("Load upgrade functions ..."); |
|||
loadSql(conn); |
|||
boolean versionValid = checkVersion(conn); |
|||
if (!versionValid) { |
|||
log.info("PostgreSQL version should be at least more than 9.6!"); |
|||
log.info("Please upgrade your PostgreSQL and restart the script!"); |
|||
} else { |
|||
log.info("PostgreSQL version is valid!"); |
|||
log.info("Updating schema ..."); |
|||
executeFunction(conn, CALL_CREATE_TS_KV_LATEST_TABLE); |
|||
executeFunction(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE); |
|||
|
|||
executeQuery(conn, "SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);"); |
|||
|
|||
executeFunction(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); |
|||
executeFunction(conn, CALL_INSERT_INTO_DICTIONARY); |
|||
executeFunction(conn, CALL_INSERT_INTO_TS_KV); |
|||
executeFunction(conn, CALL_INSERT_INTO_TS_KV_LATEST); |
|||
|
|||
//executeQuery(conn, "SELECT set_chunk_time_interval('tenant_ts_kv', " + chunkTimeInterval +");");
|
|||
|
|||
executeDropStatement(conn, DROP_OLD_TENANT_TS_KV_TABLE); |
|||
|
|||
executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_LATEST_TABLE); |
|||
executeDropStatement(conn, DROP_FUNCTION_CREATE_TENANT_TS_KV_TABLE_COPY); |
|||
executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE); |
|||
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_DICTIONARY); |
|||
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TENANT_TS_KV); |
|||
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST); |
|||
|
|||
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN json_v json;"); |
|||
executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;"); |
|||
|
|||
log.info("schema timeseries updated!"); |
|||
} |
|||
} |
|||
break; |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
protected void loadSql(Connection conn) { |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL); |
|||
try { |
|||
loadFunctions(schemaUpdateFile, conn); |
|||
log.info("Upgrade functions successfully loaded!"); |
|||
} catch (Exception e) { |
|||
log.info("Failed to load Timescale upgrade functions due to: {}", e.getMessage()); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,46 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.script; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import lombok.AllArgsConstructor; |
|||
|
|||
import javax.annotation.Nullable; |
|||
import java.util.concurrent.TimeoutException; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
@AllArgsConstructor |
|||
public class JsStatCallback<T> implements FutureCallback<T> { |
|||
|
|||
private final AtomicInteger jsSuccessMsgs; |
|||
private final AtomicInteger jsTimeoutMsgs; |
|||
private final AtomicInteger jsFailedMsgs; |
|||
|
|||
|
|||
@Override |
|||
public void onSuccess(@Nullable T result) { |
|||
jsSuccessMsgs.incrementAndGet(); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) { |
|||
jsTimeoutMsgs.incrementAndGet(); |
|||
} else { |
|||
jsFailedMsgs.incrementAndGet(); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,83 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.transport; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.gen.transport.TransportProtos; |
|||
|
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
@Slf4j |
|||
public class RuleEngineStats { |
|||
|
|||
private final AtomicInteger totalCounter = new AtomicInteger(0); |
|||
private final AtomicInteger sessionEventCounter = new AtomicInteger(0); |
|||
private final AtomicInteger postTelemetryCounter = new AtomicInteger(0); |
|||
private final AtomicInteger postAttributesCounter = new AtomicInteger(0); |
|||
private final AtomicInteger getAttributesCounter = new AtomicInteger(0); |
|||
private final AtomicInteger subscribeToAttributesCounter = new AtomicInteger(0); |
|||
private final AtomicInteger subscribeToRPCCounter = new AtomicInteger(0); |
|||
private final AtomicInteger toDeviceRPCCallResponseCounter = new AtomicInteger(0); |
|||
private final AtomicInteger toServerRPCCallRequestCounter = new AtomicInteger(0); |
|||
private final AtomicInteger subscriptionInfoCounter = new AtomicInteger(0); |
|||
private final AtomicInteger claimDeviceCounter = new AtomicInteger(0); |
|||
|
|||
public void log(TransportProtos.TransportToDeviceActorMsg msg) { |
|||
totalCounter.incrementAndGet(); |
|||
if (msg.hasSessionEvent()) { |
|||
sessionEventCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasPostTelemetry()) { |
|||
postTelemetryCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasPostAttributes()) { |
|||
postAttributesCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasGetAttributes()) { |
|||
getAttributesCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasSubscribeToAttributes()) { |
|||
subscribeToAttributesCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasSubscribeToRPC()) { |
|||
subscribeToRPCCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasToDeviceRPCCallResponse()) { |
|||
toDeviceRPCCallResponseCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasToServerRPCCallRequest()) { |
|||
toServerRPCCallRequestCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasSubscriptionInfo()) { |
|||
subscriptionInfoCounter.incrementAndGet(); |
|||
} |
|||
if (msg.hasClaimDevice()) { |
|||
claimDeviceCounter.incrementAndGet(); |
|||
} |
|||
} |
|||
|
|||
public void printStats() { |
|||
int total = totalCounter.getAndSet(0); |
|||
if (total > 0) { |
|||
log.info("Transport total [{}] sessionEvents [{}] telemetry [{}] attributes [{}] getAttr [{}] subToAttr [{}] subToRpc [{}] toDevRpc [{}] " + |
|||
"toServerRpc [{}] subInfo [{}] claimDevice [{}] ", |
|||
total, sessionEventCounter.getAndSet(0), postTelemetryCounter.getAndSet(0), |
|||
postAttributesCounter.getAndSet(0), getAttributesCounter.getAndSet(0), subscribeToAttributesCounter.getAndSet(0), |
|||
subscribeToRPCCounter.getAndSet(0), toDeviceRPCCallResponseCounter.getAndSet(0), |
|||
toServerRPCCallRequestCounter.getAndSet(0), subscriptionInfoCounter.getAndSet(0), claimDeviceCounter.getAndSet(0)); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,55 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.utils; |
|||
|
|||
import com.google.gson.JsonElement; |
|||
import com.google.gson.JsonObject; |
|||
import com.google.gson.JsonParser; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; |
|||
import java.util.List; |
|||
|
|||
public class JsonUtils { |
|||
|
|||
private static final JsonParser jsonParser = new JsonParser(); |
|||
|
|||
public static JsonObject getJsonObject(List<KeyValueProto> tsKv) { |
|||
JsonObject json = new JsonObject(); |
|||
for (KeyValueProto kv : tsKv) { |
|||
switch (kv.getType()) { |
|||
case BOOLEAN_V: |
|||
json.addProperty(kv.getKey(), kv.getBoolV()); |
|||
break; |
|||
case LONG_V: |
|||
json.addProperty(kv.getKey(), kv.getLongV()); |
|||
break; |
|||
case DOUBLE_V: |
|||
json.addProperty(kv.getKey(), kv.getDoubleV()); |
|||
break; |
|||
case STRING_V: |
|||
json.addProperty(kv.getKey(), kv.getStringV()); |
|||
break; |
|||
case JSON_V: |
|||
json.add(kv.getKey(), jsonParser.parse(kv.getJsonV())); |
|||
break; |
|||
} |
|||
} |
|||
return json; |
|||
} |
|||
|
|||
public static JsonElement parse(String params) { |
|||
return jsonParser.parse(params); |
|||
} |
|||
} |
|||
@ -0,0 +1,112 @@ |
|||
#* |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0 |
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*# |
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
|||
<html xmlns="http://www.w3.org/1999/xhtml" style="font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> |
|||
<head> |
|||
<meta name="viewport" content="width=device-width" /> |
|||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> |
|||
<title>Thingsboard - Account Lockout</title> |
|||
|
|||
|
|||
<style type="text/css"> |
|||
img { |
|||
max-width: 100%; |
|||
} |
|||
body { |
|||
-webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; |
|||
} |
|||
body { |
|||
background-color: #f6f6f6; |
|||
} |
|||
@media only screen and (max-width: 640px) { |
|||
body { |
|||
padding: 0 !important; |
|||
} |
|||
h1 { |
|||
font-weight: 800 !important; margin: 20px 0 5px !important; |
|||
} |
|||
h2 { |
|||
font-weight: 800 !important; margin: 20px 0 5px !important; |
|||
} |
|||
h3 { |
|||
font-weight: 800 !important; margin: 20px 0 5px !important; |
|||
} |
|||
h4 { |
|||
font-weight: 800 !important; margin: 20px 0 5px !important; |
|||
} |
|||
h1 { |
|||
font-size: 22px !important; |
|||
} |
|||
h2 { |
|||
font-size: 18px !important; |
|||
} |
|||
h3 { |
|||
font-size: 16px !important; |
|||
} |
|||
.container { |
|||
padding: 0 !important; width: 100% !important; |
|||
} |
|||
.content { |
|||
padding: 0 !important; |
|||
} |
|||
.content-wrap { |
|||
padding: 10px !important; |
|||
} |
|||
.invoice { |
|||
width: 100% !important; |
|||
} |
|||
} |
|||
</style> |
|||
</head> |
|||
|
|||
<body itemscope itemtype="http://schema.org/EmailMessage" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: none; width: 100% !important; height: 100%; line-height: 1.6em; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6"> |
|||
|
|||
<table class="body-wrap" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; background-color: #f6f6f6; margin: 0;" bgcolor="#f6f6f6"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" valign="top"></td> |
|||
<td class="container" width="600" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; display: block !important; max-width: 600px !important; clear: both !important; margin: 0 auto;" valign="top"> |
|||
<div class="content" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; max-width: 600px; display: block; margin: 0 auto; padding: 20px;"> |
|||
<table class="main" width="100%" cellpadding="0" cellspacing="0" itemprop="action" itemscope itemtype="http://schema.org/ConfirmAction" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; border-radius: 3px; background-color: #fff; margin: 0; border: 1px solid #e9e9e9;" bgcolor="#fff"><tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"><td class="content-wrap" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 20px;" valign="top"> |
|||
<meta itemprop="name" content="Confirm Email" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;" /><table width="100%" cellpadding="0" cellspacing="0" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> |
|||
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> |
|||
<td class="content-block" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; color: #348eda; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top"> |
|||
<h2>Thingsboard user account has been locked out</h2> |
|||
</td> |
|||
</tr> |
|||
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> |
|||
<td class="content-block" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top"> |
|||
Thingsboard user account $lockoutAccount has been lockout due to failed credentials were provided more than $maxFailedLoginAttempts times. |
|||
</td> |
|||
</tr> |
|||
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> |
|||
<td class="content-block" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top"> |
|||
— The Thingsboard |
|||
</td> |
|||
</tr></table></td> |
|||
</tr> |
|||
</table> |
|||
<div class="footer" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; width: 100%; clear: both; color: #999; margin: 0; padding: 20px;"> |
|||
<table width="100%" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> |
|||
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;"> |
|||
<td class="aligncenter content-block" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; vertical-align: top; color: #999; text-align: center; margin: 0; padding: 0 0 20px;" align="center" valign="top">This email was sent to <a href="mailto:$targetEmail" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 12px; color: #999; text-decoration: underline; margin: 0;">$targetEmail</a> by Thingsboard.</td> |
|||
</tr> |
|||
</table> |
|||
</div> |
|||
</div> |
|||
</td> |
|||
<td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0;" valign="top"></td> |
|||
</tr> |
|||
</table> |
|||
</body> |
|||
</html> |
|||
@ -0,0 +1,68 @@ |
|||
#!/bin/bash |
|||
# |
|||
# Copyright © 2016-2020 The Thingsboard Authors |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
# |
|||
|
|||
for i in "$@" |
|||
do |
|||
case $i in |
|||
--fromVersion=*) |
|||
FROM_VERSION="${i#*=}" |
|||
shift |
|||
;; |
|||
*) |
|||
# unknown option |
|||
;; |
|||
esac |
|||
done |
|||
|
|||
if [[ -z "${FROM_VERSION// }" ]]; then |
|||
echo "--fromVersion parameter is invalid or unspecified!" |
|||
echo "Usage: upgrade_dev_db.sh --fromVersion={VERSION}" |
|||
exit 1 |
|||
else |
|||
fromVersion="${FROM_VERSION// }" |
|||
fi |
|||
|
|||
BASE=${project.basedir}/target |
|||
CONF_FOLDER=${BASE}/conf |
|||
jarfile="${BASE}/thingsboard-${project.version}-boot.jar" |
|||
installDir=${BASE}/data |
|||
loadDemo=true |
|||
|
|||
|
|||
export JAVA_OPTS="$JAVA_OPTS -Dplatform=@pkg.platform@" |
|||
export LOADER_PATH=${BASE}/conf,${BASE}/extensions |
|||
export SQL_DATA_FOLDER=${SQL_DATA_FOLDER:-/tmp} |
|||
|
|||
|
|||
run_user="$USER" |
|||
|
|||
sudo -u "$run_user" -s /bin/sh -c "java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \ |
|||
-Dinstall.data_dir=${installDir} \ |
|||
-Dinstall.load_demo=${loadDemo} \ |
|||
-Dspring.jpa.hibernate.ddl-auto=none \ |
|||
-Dinstall.upgrade=true \ |
|||
-Dinstall.upgrade.from_version=${fromVersion} \ |
|||
-Dlogging.config=logback.xml \ |
|||
org.springframework.boot.loader.PropertiesLauncher" |
|||
|
|||
if [ $? -ne 0 ]; then |
|||
echo "ThingsBoard DB installation failed!" |
|||
else |
|||
echo "ThingsBoard DB installed successfully!" |
|||
fi |
|||
|
|||
exit $? |
|||
@ -0,0 +1,105 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cluster.routing; |
|||
|
|||
import com.datastax.driver.core.utils.UUIDs; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
import org.junit.runner.RunWith; |
|||
import org.mockito.runners.MockitoJUnitRunner; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.test.util.ReflectionTestUtils; |
|||
import org.thingsboard.server.common.data.Device; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.msg.cluster.ServerAddress; |
|||
import org.thingsboard.server.common.msg.cluster.ServerType; |
|||
import org.thingsboard.server.service.cluster.discovery.DiscoveryService; |
|||
import org.thingsboard.server.service.cluster.discovery.ServerInstance; |
|||
|
|||
import java.io.IOException; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Paths; |
|||
import java.util.ArrayList; |
|||
import java.util.Comparator; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.mockito.Mockito.mock; |
|||
import static org.mockito.Mockito.when; |
|||
|
|||
@Slf4j |
|||
@RunWith(MockitoJUnitRunner.class) |
|||
public class ConsistentClusterRoutingServiceTest { |
|||
|
|||
private ConsistentClusterRoutingService clusterRoutingService; |
|||
|
|||
private DiscoveryService discoveryService; |
|||
|
|||
private String hashFunctionName = "murmur3_128"; |
|||
private Integer virtualNodesSize = 1024*4; |
|||
private ServerAddress currentServer = new ServerAddress(" 100.96.1.0", 9001, ServerType.CORE); |
|||
|
|||
|
|||
@Before |
|||
public void setup() throws Exception { |
|||
discoveryService = mock(DiscoveryService.class); |
|||
clusterRoutingService = new ConsistentClusterRoutingService(); |
|||
ReflectionTestUtils.setField(clusterRoutingService, "discoveryService", discoveryService); |
|||
ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); |
|||
ReflectionTestUtils.setField(clusterRoutingService, "virtualNodesSize", virtualNodesSize); |
|||
when(discoveryService.getCurrentServer()).thenReturn(new ServerInstance(currentServer)); |
|||
List<ServerInstance> otherServers = new ArrayList<>(); |
|||
for (int i = 1; i < 30; i++) { |
|||
otherServers.add(new ServerInstance(new ServerAddress(" 100.96." + i*2 + "." + i, 9001, ServerType.CORE))); |
|||
} |
|||
when(discoveryService.getOtherServers()).thenReturn(otherServers); |
|||
clusterRoutingService.init(); |
|||
} |
|||
|
|||
@Test |
|||
public void testDispersionOnMillionDevices() { |
|||
List<DeviceId> devices = new ArrayList<>(); |
|||
for (int i = 0; i < 1000000; i++) { |
|||
devices.add(new DeviceId(UUIDs.timeBased())); |
|||
} |
|||
|
|||
testDevicesDispersion(devices); |
|||
} |
|||
|
|||
private void testDevicesDispersion(List<DeviceId> devices) { |
|||
long start = System.currentTimeMillis(); |
|||
Map<ServerAddress, Integer> map = new HashMap<>(); |
|||
for (DeviceId deviceId : devices) { |
|||
ServerAddress address = clusterRoutingService.resolveById(deviceId).orElse(currentServer); |
|||
map.put(address, map.getOrDefault(address, 0) + 1); |
|||
} |
|||
|
|||
List<Map.Entry<ServerAddress, Integer>> data = map.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getValue)).collect(Collectors.toList()); |
|||
long end = System.currentTimeMillis(); |
|||
System.out.println("Size: " + virtualNodesSize + " Time: " + (end - start) + " Diff: " + (data.get(data.size() - 1).getValue() - data.get(0).getValue())); |
|||
|
|||
for (Map.Entry<ServerAddress, Integer> entry : data) { |
|||
// System.out.println(entry.getKey().getHost() + ": " + entry.getValue());
|
|||
} |
|||
|
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
|
|||
@ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.HSQLDialect") |
|||
public @interface HsqlDao { |
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
|
|||
@ConditionalOnProperty(prefix = "spring.jpa", value = "database-platform", havingValue = "org.hibernate.dialect.PostgreSQLDialect") |
|||
public @interface PsqlDao { |
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
|
|||
@ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + |
|||
"&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") |
|||
public @interface PsqlTsAnyDao { |
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
|
|||
@ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") |
|||
public @interface SqlTsAnyDao { |
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
|
|||
@ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "timescale") |
|||
public @interface TimescaleDBTsDao { |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue