Browse Source

Merge with master

pull/2611/head
Andrii Shvaika 6 years ago
parent
commit
2f6c6ad1fe
  1. 123
      application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql
  2. 96
      application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql
  3. 128
      application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql
  4. 125
      application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql
  5. 5
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  6. 1
      application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java
  7. 8
      application/src/main/java/org/thingsboard/server/controller/AlarmController.java
  8. 7
      application/src/main/java/org/thingsboard/server/controller/DeviceController.java
  9. 33
      application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java
  10. 3
      application/src/main/java/org/thingsboard/server/controller/EntityViewController.java
  11. 17
      application/src/main/java/org/thingsboard/server/controller/TelemetryController.java
  12. 95
      application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java
  13. 87
      application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
  14. 4
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java
  15. 95
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
  16. 8
      application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java
  17. 9
      application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
  18. 7
      application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java
  19. 8
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  20. 14
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java
  21. 11
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  22. 89
      application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java
  23. 41
      application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java
  24. 35
      application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java
  25. 4
      application/src/main/resources/thingsboard.yml
  26. 4
      application/src/main/scripts/control/deb/postinst
  27. 10
      application/src/main/scripts/control/deb/preinst
  28. 4
      application/src/main/scripts/control/rpm/postinst
  29. 6
      application/src/main/scripts/control/rpm/preinst
  30. 2
      application/src/main/scripts/control/thingsboard.service
  31. 2
      application/src/main/scripts/install/install.sh
  32. 2
      application/src/main/scripts/install/upgrade.sh
  33. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java
  34. 21
      common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java
  35. 3
      common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java
  36. 2
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java
  37. 11
      common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java
  38. 13
      dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java
  39. 13
      dao/src/main/java/org/thingsboard/server/dao/alarm/CassandraAlarmDao.java
  40. 34
      dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java
  41. 3
      dao/src/main/java/org/thingsboard/server/dao/asset/CassandraAssetDao.java
  42. 3
      dao/src/main/java/org/thingsboard/server/dao/dashboard/CassandraDashboardInfoDao.java
  43. 37
      dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java
  44. 5
      dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java
  45. 9
      dao/src/main/java/org/thingsboard/server/dao/device/ClaimDevicesServiceImpl.java
  46. 27
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
  47. 5
      dao/src/main/java/org/thingsboard/server/dao/entityview/CassandraEntityViewDao.java
  48. 9
      dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java
  49. 11
      dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java
  50. 5
      dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java
  51. 4
      dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java
  52. 1
      dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvCompositeKey.java
  53. 19
      dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java
  54. 4
      dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java
  55. 7
      dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java
  56. 7
      dao/src/main/java/org/thingsboard/server/dao/nosql/RateLimitedResultSetFuture.java
  57. 5
      dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java
  58. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java
  59. 39
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
  60. 21
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java
  61. 43
      dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java
  62. 25
      dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java
  63. 45
      dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java
  64. 10
      dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java
  65. 17
      dao/src/main/resources/sql/schema-timescale-idx.sql
  66. 125
      dao/src/main/resources/sql/schema-timescale.sql
  67. 271
      dao/src/main/resources/sql/schema-ts-psql.sql
  68. 2
      dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java
  69. 8
      dao/src/test/java/org/thingsboard/server/dao/nosql/RateLimitedResultSetFutureTest.java
  70. 2
      dao/src/test/resources/sql/timescale/drop-all-tables.sql
  71. 7
      docker/README.md
  72. 24
      docker/docker-create-log-folders.sh
  73. 2
      k8s/database-setup.yml
  74. 4
      k8s/k8s-delete-all.sh
  75. 2
      k8s/k8s-install-tb.sh
  76. 2
      k8s/k8s-upgrade-tb.sh
  77. 2
      k8s/postgres.yml
  78. 4
      msa/js-executor/docker/Dockerfile
  79. 4
      msa/js-executor/docker/start-js-executor.sh
  80. 1
      msa/js-executor/pom.xml
  81. 4
      msa/tb-node/docker/Dockerfile
  82. 4
      msa/tb-node/docker/start-tb-node.sh
  83. 1
      msa/tb-node/pom.xml
  84. 1
      msa/tb/pom.xml
  85. 2
      msa/transport/coap/docker/Dockerfile
  86. 2
      msa/transport/coap/docker/start-tb-coap-transport.sh
  87. 1
      msa/transport/coap/pom.xml
  88. 2
      msa/transport/http/docker/Dockerfile
  89. 2
      msa/transport/http/docker/start-tb-http-transport.sh
  90. 1
      msa/transport/http/pom.xml
  91. 2
      msa/transport/mqtt/docker/Dockerfile
  92. 2
      msa/transport/mqtt/docker/start-tb-mqtt-transport.sh
  93. 1
      msa/transport/mqtt/pom.xml
  94. 4
      msa/web-ui/docker/Dockerfile
  95. 4
      msa/web-ui/docker/start-web-ui.sh
  96. 1
      msa/web-ui/pom.xml
  97. 8
      netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java
  98. 5
      pom.xml
  99. 5
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java
  100. 7
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java

123
application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql

@ -0,0 +1,123 @@
--
-- Copyright © 2016-2020 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
max_tenant_ttl bigint;
max_customer_ttl bigint;
max_ttl bigint;
date timestamp;
partition_by_max_ttl_date varchar;
partition_month varchar;
partition_day varchar;
partition_year varchar;
partition varchar;
partition_to_delete varchar;
BEGIN
SELECT max(attribute_kv.long_v)
FROM tenant
INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id
WHERE attribute_kv.attribute_key = 'TTL'
into max_tenant_ttl;
SELECT max(attribute_kv.long_v)
FROM customer
INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id
WHERE attribute_kv.attribute_key = 'TTL'
into max_customer_ttl;
max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
if max_ttl IS NOT NULL AND max_ttl > 0 THEN
date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
IF partition_by_max_ttl_date IS NOT NULL THEN
CASE
WHEN partition_type = 'DAYS' THEN
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
WHEN partition_type = 'MONTHS' THEN
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
ELSE
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
END CASE;
FOR partition IN SELECT tablename
FROM pg_tables
WHERE schemaname = 'public'
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
LOOP
IF partition != partition_by_max_ttl_date THEN
IF partition_year IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN
partition_to_delete := partition;
ELSE
IF partition_month IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN
partition_to_delete := partition;
ELSE
IF partition_day IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN
partition_to_delete := partition;
END IF;
END IF;
END IF;
END IF;
END IF;
END IF;
END IF;
IF partition_to_delete IS NOT NULL THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
EXECUTE format('DROP TABLE %I', partition_to_delete);
deleted := deleted + 1;
END IF;
END LOOP;
END IF;
END IF;
END
$$;
CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS
$$
BEGIN
CASE
WHEN partition_type = 'DAYS' THEN
partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd');
WHEN partition_type = 'MONTHS' THEN
partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM');
WHEN partition_type = 'YEARS' THEN
partition := 'ts_kv_' || to_char(date, 'yyyy');
WHEN partition_type = 'INDEFINITE' THEN
partition := NULL;
ELSE
partition := NULL;
END CASE;
IF partition IS NOT NULL THEN
IF NOT EXISTS(SELECT
FROM pg_tables
WHERE schemaname = 'public'
AND tablename = partition) THEN
partition := NULL;
RAISE NOTICE 'Failed to found partition by ttl';
END IF;
END IF;
END;
$$ LANGUAGE plpgsql;

96
application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql

@ -14,33 +14,9 @@
-- limitations under the License.
--
-- select check_version();
-- call create_partition_ts_kv_table();
CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$
DECLARE
current_version integer;
valid_version boolean;
BEGIN
RAISE NOTICE 'Check the current installed PostgreSQL version...';
SELECT current_setting('server_version_num') INTO current_version;
IF current_version < 100000 THEN
valid_version := FALSE;
ELSE
valid_version := TRUE;
END IF;
IF valid_version = FALSE THEN
RAISE NOTICE 'Postgres version should be at least more than 10!';
ELSE
RAISE NOTICE 'PostgreSQL version is valid!';
RAISE NOTICE 'Schema update started...';
END IF;
RETURN valid_version;
END;
$$ LANGUAGE 'plpgsql';
-- select create_partition_ts_kv_table();
CREATE OR REPLACE FUNCTION create_partition_ts_kv_table() RETURNS VOID AS $$
CREATE OR REPLACE PROCEDURE create_partition_ts_kv_table() LANGUAGE plpgsql AS $$
BEGIN
ALTER TABLE ts_kv
@ -57,11 +33,11 @@ BEGIN
ALTER TABLE ts_kv
ALTER COLUMN key TYPE integer USING key::integer;
END;
$$ LANGUAGE 'plpgsql';
$$;
-- select create_new_ts_kv_latest_table();
-- call create_new_ts_kv_latest_table();
CREATE OR REPLACE FUNCTION create_new_ts_kv_latest_table() RETURNS VOID AS $$
CREATE OR REPLACE PROCEDURE create_new_ts_kv_latest_table() LANGUAGE plpgsql AS $$
BEGIN
ALTER TABLE ts_kv_latest
@ -81,13 +57,13 @@ BEGIN
ALTER TABLE ts_kv_latest
ADD CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key);
END;
$$ LANGUAGE 'plpgsql';
$$;
-- call create_partitions();
-- select create_partitions();
CREATE OR REPLACE PROCEDURE create_partitions() LANGUAGE plpgsql AS $$
CREATE OR REPLACE FUNCTION create_partitions() RETURNS VOID AS
$$
DECLARE
partition_date varchar;
from_ts bigint;
@ -111,11 +87,11 @@ BEGIN
CLOSE key_cursor;
END;
$$ language 'plpgsql';
$$;
-- select create_ts_kv_dictionary_table();
-- call create_ts_kv_dictionary_table();
CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$
CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table() LANGUAGE plpgsql AS $$
BEGIN
CREATE TABLE IF NOT EXISTS ts_kv_dictionary
@ -125,12 +101,12 @@ BEGIN
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
);
END;
$$ LANGUAGE 'plpgsql';
$$;
-- call insert_into_dictionary();
-- select insert_into_dictionary();
CREATE OR REPLACE PROCEDURE insert_into_dictionary() LANGUAGE plpgsql AS $$
CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS
$$
DECLARE
insert_record RECORD;
key_cursor CURSOR FOR SELECT DISTINCT key
@ -150,28 +126,27 @@ BEGIN
END LOOP;
CLOSE key_cursor;
END;
$$ language 'plpgsql';
$$;
-- select insert_into_ts_kv();
-- call insert_into_ts_kv();
CREATE OR REPLACE FUNCTION insert_into_ts_kv() RETURNS void AS
$$
CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(first_part_uuid, '-', second_part_uuid, '-1', third_part_uuid, '-', fourth_part_uuid, '-', fifth_part_uuid)::uuid AS entity_id,
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
ts_kv_records.key AS key,
ts_kv_records.ts AS ts,
ts_kv_records.bool_v AS bool_v,
ts_kv_records.str_v AS str_v,
ts_kv_records.long_v AS long_v,
ts_kv_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first_part_uuid,
SUBSTRING(entity_id, 4, 4) AS second_part_uuid,
SUBSTRING(entity_id, 1, 3) AS third_part_uuid,
SUBSTRING(entity_id, 16, 4) AS fourth_part_uuid,
SUBSTRING(entity_id, 20) AS fifth_part_uuid,
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
key_id AS key,
ts,
bool_v,
@ -198,28 +173,27 @@ BEGIN
END LOOP;
CLOSE insert_cursor;
END;
$$ LANGUAGE 'plpgsql';
$$;
-- select insert_into_ts_kv_latest();
-- call insert_into_ts_kv_latest();
CREATE OR REPLACE FUNCTION insert_into_ts_kv_latest() RETURNS void AS
$$
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(first_part_uuid, '-', second_part_uuid, '-1', third_part_uuid, '-', fourth_part_uuid, '-', fifth_part_uuid)::uuid AS entity_id,
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
ts_kv_latest_records.key AS key,
ts_kv_latest_records.ts AS ts,
ts_kv_latest_records.bool_v AS bool_v,
ts_kv_latest_records.str_v AS str_v,
ts_kv_latest_records.long_v AS long_v,
ts_kv_latest_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS first_part_uuid,
SUBSTRING(entity_id, 4, 4) AS second_part_uuid,
SUBSTRING(entity_id, 1, 3) AS third_part_uuid,
SUBSTRING(entity_id, 16, 4) AS fourth_part_uuid,
SUBSTRING(entity_id, 20) AS fifth_part_uuid,
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
key_id AS key,
ts,
bool_v,
@ -246,6 +220,6 @@ BEGIN
END LOOP;
CLOSE insert_cursor;
END;
$$ LANGUAGE 'plpgsql';
$$;

128
application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql

@ -14,60 +14,32 @@
-- limitations under the License.
--
-- select check_version();
-- call create_new_ts_kv_table();
CREATE OR REPLACE FUNCTION check_version() RETURNS boolean AS $$
DECLARE
current_version integer;
valid_version boolean;
BEGIN
RAISE NOTICE 'Check the current installed PostgreSQL version...';
SELECT current_setting('server_version_num') INTO current_version;
IF current_version < 90600 THEN
valid_version := FALSE;
ELSE
valid_version := TRUE;
END IF;
IF valid_version = FALSE THEN
RAISE NOTICE 'Postgres version should be at least more than 9.6!';
ELSE
RAISE NOTICE 'PostgreSQL version is valid!';
RAISE NOTICE 'Schema update started...';
END IF;
RETURN valid_version;
END;
$$ LANGUAGE 'plpgsql';
-- select create_new_tenant_ts_kv_table();
CREATE OR REPLACE FUNCTION create_new_tenant_ts_kv_table() RETURNS VOID AS $$
CREATE OR REPLACE PROCEDURE create_new_ts_kv_table() LANGUAGE plpgsql AS $$
BEGIN
ALTER TABLE tenant_ts_kv
RENAME TO tenant_ts_kv_old;
CREATE TABLE IF NOT EXISTS tenant_ts_kv
CREATE TABLE IF NOT EXISTS ts_kv
(
LIKE tenant_ts_kv_old
);
ALTER TABLE tenant_ts_kv
ALTER COLUMN tenant_id TYPE uuid USING tenant_id::uuid;
ALTER TABLE tenant_ts_kv
ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
ALTER TABLE tenant_ts_kv
ALTER COLUMN key TYPE integer USING key::integer;
ALTER TABLE tenant_ts_kv
ADD CONSTRAINT tenant_ts_kv_pkey PRIMARY KEY(tenant_id, entity_id, key, ts);
ALTER TABLE ts_kv ALTER COLUMN entity_id TYPE uuid USING entity_id::uuid;
ALTER TABLE ts_kv ALTER COLUMN key TYPE integer USING key::integer;
ALTER INDEX ts_kv_pkey RENAME TO tenant_ts_kv_pkey_old;
ALTER INDEX idx_tenant_ts_kv RENAME TO idx_tenant_ts_kv_old;
ALTER INDEX tenant_ts_kv_ts_idx RENAME TO tenant_ts_kv_ts_idx_old;
-- PERFORM create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => 86400000, if_not_exists => true);
CREATE INDEX IF NOT EXISTS idx_tenant_ts_kv ON tenant_ts_kv(tenant_id, entity_id, key, ts);
ALTER TABLE ts_kv ADD CONSTRAINT ts_kv_pkey PRIMARY KEY(entity_id, key, ts);
-- CREATE INDEX IF NOT EXISTS ts_kv_ts_idx ON ts_kv(ts DESC);
ALTER TABLE ts_kv DROP COLUMN IF EXISTS tenant_id;
END;
$$ LANGUAGE 'plpgsql';
$$;
-- select create_ts_kv_latest_table();
-- call create_ts_kv_latest_table();
CREATE OR REPLACE FUNCTION create_ts_kv_latest_table() RETURNS VOID AS $$
CREATE OR REPLACE PROCEDURE create_ts_kv_latest_table() LANGUAGE plpgsql AS $$
BEGIN
CREATE TABLE IF NOT EXISTS ts_kv_latest
@ -82,12 +54,12 @@ BEGIN
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
);
END;
$$ LANGUAGE 'plpgsql';
$$;
-- select create_ts_kv_dictionary_table();
-- call create_ts_kv_dictionary_table();
CREATE OR REPLACE FUNCTION create_ts_kv_dictionary_table() RETURNS VOID AS $$
CREATE OR REPLACE PROCEDURE create_ts_kv_dictionary_table() LANGUAGE plpgsql AS $$
BEGIN
CREATE TABLE IF NOT EXISTS ts_kv_dictionary
@ -97,12 +69,12 @@ BEGIN
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
);
END;
$$ LANGUAGE 'plpgsql';
$$;
-- select insert_into_dictionary();
-- call insert_into_dictionary();
CREATE OR REPLACE PROCEDURE insert_into_dictionary() LANGUAGE plpgsql AS $$
CREATE OR REPLACE FUNCTION insert_into_dictionary() RETURNS VOID AS
$$
DECLARE
insert_record RECORD;
key_cursor CURSOR FOR SELECT DISTINCT key
@ -122,34 +94,28 @@ BEGIN
END LOOP;
CLOSE key_cursor;
END;
$$ language 'plpgsql';
$$;
-- call insert_into_ts_kv();
-- select insert_into_tenant_ts_kv();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$
CREATE OR REPLACE FUNCTION insert_into_tenant_ts_kv() RETURNS void AS
$$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(tenant_id_first_part_uuid, '-', tenant_id_second_part_uuid, '-1', tenant_id_third_part_uuid, '-', tenant_id_fourth_part_uuid, '-', tenant_id_fifth_part_uuid)::uuid AS tenant_id,
CONCAT(entity_id_first_part_uuid, '-', entity_id_second_part_uuid, '-1', entity_id_third_part_uuid, '-', entity_id_fourth_part_uuid, '-', entity_id_fifth_part_uuid)::uuid AS entity_id,
tenant_ts_kv_records.key AS key,
tenant_ts_kv_records.ts AS ts,
tenant_ts_kv_records.bool_v AS bool_v,
tenant_ts_kv_records.str_v AS str_v,
tenant_ts_kv_records.long_v AS long_v,
tenant_ts_kv_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(tenant_id, 8, 8) AS tenant_id_first_part_uuid,
SUBSTRING(tenant_id, 4, 4) AS tenant_id_second_part_uuid,
SUBSTRING(tenant_id, 1, 3) AS tenant_id_third_part_uuid,
SUBSTRING(tenant_id, 16, 4) AS tenant_id_fourth_part_uuid,
SUBSTRING(tenant_id, 20) AS tenant_id_fifth_part_uuid,
SUBSTRING(entity_id, 8, 8) AS entity_id_first_part_uuid,
SUBSTRING(entity_id, 4, 4) AS entity_id_second_part_uuid,
SUBSTRING(entity_id, 1, 3) AS entity_id_third_part_uuid,
SUBSTRING(entity_id, 16, 4) AS entity_id_fourth_part_uuid,
SUBSTRING(entity_id, 20) AS entity_id_fifth_part_uuid,
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
new_ts_kv_records.key AS key,
new_ts_kv_records.ts AS ts,
new_ts_kv_records.bool_v AS bool_v,
new_ts_kv_records.str_v AS str_v,
new_ts_kv_records.long_v AS long_v,
new_ts_kv_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
key_id AS key,
ts,
bool_v,
@ -157,31 +123,31 @@ DECLARE
long_v,
dbl_v
FROM tenant_ts_kv_old
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS tenant_ts_kv_records;
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records;
BEGIN
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter - 1;
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO tenant_ts_kv(tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.tenant_id, insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the new tenant_ts_kv table!',insert_counter;
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
END;
$$ LANGUAGE 'plpgsql';
$$;
-- call insert_into_ts_kv_latest();
-- select insert_into_ts_kv_latest();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$
CREATE OR REPLACE FUNCTION insert_into_ts_kv_latest() RETURNS void AS
$$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
@ -191,7 +157,7 @@ DECLARE
latest_records.key AS key,
latest_records.entity_id AS entity_id,
latest_records.ts AS ts
FROM (SELECT DISTINCT key AS key, entity_id AS entity_id, MAX(ts) AS ts FROM tenant_ts_kv GROUP BY key, entity_id) AS latest_records;
FROM (SELECT DISTINCT key AS key, entity_id AS entity_id, MAX(ts) AS ts FROM ts_kv GROUP BY key, entity_id) AS latest_records;
BEGIN
OPEN insert_cursor;
LOOP
@ -201,7 +167,7 @@ BEGIN
RAISE NOTICE '% records have been inserted into the ts_kv_latest table!',insert_counter - 1;
EXIT;
END IF;
SELECT entity_id AS entity_id, key AS key, ts AS ts, bool_v AS bool_v, str_v AS str_v, long_v AS long_v, dbl_v AS dbl_v INTO insert_record FROM tenant_ts_kv WHERE entity_id = latest_record.entity_id AND key = latest_record.key AND ts = latest_record.ts;
SELECT entity_id AS entity_id, key AS key, ts AS ts, bool_v AS bool_v, str_v AS str_v, long_v AS long_v, dbl_v AS dbl_v INTO insert_record FROM ts_kv WHERE entity_id = latest_record.entity_id AND key = latest_record.key AND ts = latest_record.ts;
INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
@ -210,4 +176,4 @@ BEGIN
END LOOP;
CLOSE insert_cursor;
END;
$$ LANGUAGE 'plpgsql';
$$;

125
application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql

@ -0,0 +1,125 @@
--
-- Copyright © 2016-2020 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$
BEGIN
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31),
IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
tenant_cursor CURSOR FOR select tenant.id as tenant_id
from tenant;
tenant_id_record varchar;
customer_id_record varchar;
tenant_ttl bigint;
customer_ttl bigint;
deleted_for_entities bigint;
tenant_ttl_ts bigint;
customer_ttl_ts bigint;
BEGIN
OPEN tenant_cursor;
FETCH tenant_cursor INTO tenant_id_record;
WHILE FOUND
LOOP
EXECUTE format(
'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
tenant_id_record, 'TTL') INTO tenant_ttl;
if tenant_ttl IS NULL THEN
tenant_ttl := system_ttl;
END IF;
IF tenant_ttl > 0 THEN
tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint;
deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record;
deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record;
END IF;
FOR customer_id_record IN
SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record
LOOP
EXECUTE format(
'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
customer_id_record, 'TTL') INTO customer_ttl;
IF customer_ttl IS NULL THEN
customer_ttl_ts := tenant_ttl_ts;
ELSE
IF customer_ttl > 0 THEN
customer_ttl_ts :=
(EXTRACT(EPOCH FROM current_timestamp) * 1000 -
customer_ttl::bigint * 1000)::bigint;
END IF;
END IF;
IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN
deleted_for_entities :=
delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record;
deleted_for_entities :=
delete_device_records_from_ts_kv(tenant_id_record, customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record,
customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
END IF;
END LOOP;
FETCH tenant_cursor INTO tenant_id_record;
END LOOP;
END
$$;

5
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import lombok.Getter;
@ -474,7 +475,7 @@ public class ActorSystemContext {
public void onFailure(Throwable th) {
log.error("Could not save debug Event for Node", th);
}
});
}, MoreExecutors.directExecutor());
} catch (IOException ex) {
log.warn("Failed to persist rule node debug message", ex);
}
@ -527,7 +528,7 @@ public class ActorSystemContext {
public void onFailure(Throwable th) {
log.error("Could not save debug Event for Rule Chain", th);
}
});
}, MoreExecutors.directExecutor());
}
public static Exception toException(Throwable error) {

1
application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java

@ -107,7 +107,6 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt
return filter;
}
@Bean
protected JwtTokenAuthenticationProcessingFilter buildJwtTokenAuthenticationProcessingFilter() throws Exception {
List<String> pathsToSkip = new ArrayList(Arrays.asList(NON_TOKEN_BASED_AUTH_ENTRY_POINTS));
pathsToSkip.addAll(Arrays.asList(WS_TOKEN_BASED_AUTH_ENTRY_POINT, TOKEN_REFRESH_ENTRY_POINT, FORM_BASED_LOGIN_ENTRY_POINT,

8
application/src/main/java/org/thingsboard/server/controller/AlarmController.java

@ -120,7 +120,9 @@ public class AlarmController extends BaseController {
try {
AlarmId alarmId = new AlarmId(toUUID(strAlarmId));
Alarm alarm = checkAlarmId(alarmId, Operation.WRITE);
alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, System.currentTimeMillis()).get();
long ackTs = System.currentTimeMillis();
alarmService.ackAlarm(getCurrentUser().getTenantId(), alarmId, ackTs).get();
alarm.setAckTs(ackTs);
logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_ACK, null);
} catch (Exception e) {
throw handleException(e);
@ -135,7 +137,9 @@ public class AlarmController extends BaseController {
try {
AlarmId alarmId = new AlarmId(toUUID(strAlarmId));
Alarm alarm = checkAlarmId(alarmId, Operation.WRITE);
alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, System.currentTimeMillis()).get();
long clearTs = System.currentTimeMillis();
alarmService.clearAlarm(getCurrentUser().getTenantId(), alarmId, null, clearTs).get();
alarm.setClearTs(clearTs);
logEntityAction(alarmId, alarm, getCurrentUser().getCustomerId(), ActionType.ALARM_CLEAR, null);
} catch (Exception e) {
throw handleException(e);

7
application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@ -18,6 +18,7 @@ package org.thingsboard.server.controller;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
@ -32,6 +33,7 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg;
import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.common.data.ClaimRequest;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
@ -46,7 +48,6 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.TextPageData;
import org.thingsboard.server.common.data.page.TextPageLink;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.ClaimRequest;
import org.thingsboard.server.dao.device.claim.ClaimResponse;
import org.thingsboard.server.dao.device.claim.ClaimResult;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
@ -432,7 +433,7 @@ public class DeviceController extends BaseController {
public void onFailure(Throwable t) {
deferredResult.setErrorResult(t);
}
});
}, MoreExecutors.directExecutor());
return deferredResult;
} catch (Exception e) {
throw handleException(e);
@ -469,7 +470,7 @@ public class DeviceController extends BaseController {
public void onFailure(Throwable t) {
deferredResult.setErrorResult(t);
}
});
}, MoreExecutors.directExecutor());
return deferredResult;
} catch (Exception e) {
throw handleException(e);

33
application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java

@ -37,6 +37,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.permission.Operation;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@ -167,7 +168,7 @@ public class EntityRelationController extends BaseController {
checkEntityId(entityId, Operation.READ);
RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
try {
return checkNotNull(relationService.findByFrom(getTenantId(), entityId, typeGroup));
return checkNotNull(filterRelationsByReadPermission(relationService.findByFrom(getTenantId(), entityId, typeGroup)));
} catch (Exception e) {
throw handleException(e);
}
@ -185,7 +186,7 @@ public class EntityRelationController extends BaseController {
checkEntityId(entityId, Operation.READ);
RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
try {
return checkNotNull(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get());
return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByFrom(getTenantId(), entityId, typeGroup).get()));
} catch (Exception e) {
throw handleException(e);
}
@ -205,7 +206,7 @@ public class EntityRelationController extends BaseController {
checkEntityId(entityId, Operation.READ);
RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
try {
return checkNotNull(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup));
return checkNotNull(filterRelationsByReadPermission(relationService.findByFromAndType(getTenantId(), entityId, strRelationType, typeGroup)));
} catch (Exception e) {
throw handleException(e);
}
@ -223,7 +224,7 @@ public class EntityRelationController extends BaseController {
checkEntityId(entityId, Operation.READ);
RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
try {
return checkNotNull(relationService.findByTo(getTenantId(), entityId, typeGroup));
return checkNotNull(filterRelationsByReadPermission(relationService.findByTo(getTenantId(), entityId, typeGroup)));
} catch (Exception e) {
throw handleException(e);
}
@ -241,7 +242,7 @@ public class EntityRelationController extends BaseController {
checkEntityId(entityId, Operation.READ);
RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
try {
return checkNotNull(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get());
return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByTo(getTenantId(), entityId, typeGroup).get()));
} catch (Exception e) {
throw handleException(e);
}
@ -261,7 +262,7 @@ public class EntityRelationController extends BaseController {
checkEntityId(entityId, Operation.READ);
RelationTypeGroup typeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON);
try {
return checkNotNull(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup));
return checkNotNull(filterRelationsByReadPermission(relationService.findByToAndType(getTenantId(), entityId, strRelationType, typeGroup)));
} catch (Exception e) {
throw handleException(e);
}
@ -276,7 +277,7 @@ public class EntityRelationController extends BaseController {
checkNotNull(query.getFilters());
checkEntityId(query.getParameters().getEntityId(), Operation.READ);
try {
return checkNotNull(relationService.findByQuery(getTenantId(), query).get());
return checkNotNull(filterRelationsByReadPermission(relationService.findByQuery(getTenantId(), query).get()));
} catch (Exception e) {
throw handleException(e);
}
@ -291,12 +292,28 @@ public class EntityRelationController extends BaseController {
checkNotNull(query.getFilters());
checkEntityId(query.getParameters().getEntityId(), Operation.READ);
try {
return checkNotNull(relationService.findInfoByQuery(getTenantId(), query).get());
return checkNotNull(filterRelationsByReadPermission(relationService.findInfoByQuery(getTenantId(), query).get()));
} catch (Exception e) {
throw handleException(e);
}
}
private <T extends EntityRelation> List<T> filterRelationsByReadPermission(List<T> relationsByQuery) {
return relationsByQuery.stream().filter(relationByQuery -> {
try {
checkEntityId(relationByQuery.getTo(), Operation.READ);
} catch (ThingsboardException e) {
return false;
}
try {
checkEntityId(relationByQuery.getFrom(), Operation.READ);
} catch (ThingsboardException e) {
return false;
}
return true;
}).collect(Collectors.toList());
}
private RelationTypeGroup parseRelationTypeGroup(String strRelationTypeGroup, RelationTypeGroup defaultValue) {
RelationTypeGroup result = defaultValue;
if (strRelationTypeGroup != null && strRelationTypeGroup.trim().length() > 0) {

3
application/src/main/java/org/thingsboard/server/controller/EntityViewController.java

@ -18,6 +18,7 @@ package org.thingsboard.server.controller;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.security.access.prepost.PreAuthorize;
@ -160,7 +161,7 @@ public class EntityViewController extends BaseController {
});
}
return null;
});
}, MoreExecutors.directExecutor());
} else {
return Futures.immediateFuture(null);
}

17
application/src/main/java/org/thingsboard/server/controller/TelemetryController.java

@ -22,6 +22,7 @@ import com.google.common.base.Function;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
@ -175,7 +176,7 @@ public class TelemetryController extends BaseController {
public DeferredResult<ResponseEntity> getTimeseriesKeys(
@PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr) throws ThingsboardException {
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr,
(result, tenantId, entityId) -> Futures.addCallback(tsService.findAllLatest(tenantId, entityId), getTsKeysToResponseCallback(result)));
(result, tenantId, entityId) -> Futures.addCallback(tsService.findAllLatest(tenantId, entityId), getTsKeysToResponseCallback(result), MoreExecutors.directExecutor()));
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@ -211,7 +212,7 @@ public class TelemetryController extends BaseController {
List<ReadTsKvQuery> queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg))
.collect(Collectors.toList());
Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes));
Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor());
});
}
@ -461,7 +462,7 @@ public class TelemetryController extends BaseController {
} else {
future = tsService.findLatest(user.getTenantId(), entityId, toKeysList(keys));
}
Futures.addCallback(future, getTsKvListCallback(result, useStrictDataTypes));
Futures.addCallback(future, getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor());
}
private void getAttributeValuesCallback(@Nullable DeferredResult<ResponseEntity> result, SecurityUser user, EntityId entityId, String scope, String keys) {
@ -469,9 +470,9 @@ public class TelemetryController extends BaseController {
FutureCallback<List<AttributeKvEntry>> callback = getAttributeValuesToResponseCallback(result, user, scope, entityId, keyList);
if (!StringUtils.isEmpty(scope)) {
if (keyList != null && !keyList.isEmpty()) {
Futures.addCallback(attributesService.find(user.getTenantId(), entityId, scope, keyList), callback);
Futures.addCallback(attributesService.find(user.getTenantId(), entityId, scope, keyList), callback, MoreExecutors.directExecutor());
} else {
Futures.addCallback(attributesService.findAll(user.getTenantId(), entityId, scope), callback);
Futures.addCallback(attributesService.findAll(user.getTenantId(), entityId, scope), callback, MoreExecutors.directExecutor());
}
} else {
List<ListenableFuture<List<AttributeKvEntry>>> futures = new ArrayList<>();
@ -485,12 +486,12 @@ public class TelemetryController extends BaseController {
ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
Futures.addCallback(future, callback);
Futures.addCallback(future, callback, MoreExecutors.directExecutor());
}
}
private void getAttributeKeysCallback(@Nullable DeferredResult<ResponseEntity> result, TenantId tenantId, EntityId entityId, String scope) {
Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), getAttributeKeysToResponseCallback(result));
Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), getAttributeKeysToResponseCallback(result), MoreExecutors.directExecutor());
}
private void getAttributeKeysCallback(@Nullable DeferredResult<ResponseEntity> result, TenantId tenantId, EntityId entityId) {
@ -501,7 +502,7 @@ public class TelemetryController extends BaseController {
ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
Futures.addCallback(future, getAttributeKeysToResponseCallback(result));
Futures.addCallback(future, getAttributeKeysToResponseCallback(result), MoreExecutors.directExecutor());
}
private FutureCallback<List<TsKvEntry>> getTsKeysToResponseCallback(final DeferredResult<ResponseEntity> response) {

95
application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java

@ -22,38 +22,18 @@ import org.springframework.beans.factory.annotation.Value;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.sql.Types;
import java.sql.Statement;
@Slf4j
public abstract class AbstractSqlTsDatabaseUpgradeService {
protected static final String CALL_REGEX = "call ";
protected static final String CHECK_VERSION = "check_version()";
protected static final String DROP_TABLE = "DROP TABLE ";
protected static final String DROP_FUNCTION_IF_EXISTS = "DROP FUNCTION IF EXISTS ";
private static final String CALL_CHECK_VERSION = CALL_REGEX + CHECK_VERSION;
private static final String FUNCTION = "function: {}";
private static final String DROP_STATEMENT = "drop statement: {}";
private static final String QUERY = "query: {}";
private static final String SUCCESSFULLY_EXECUTED = "Successfully executed ";
private static final String FAILED_TO_EXECUTE = "Failed to execute ";
private static final String FAILED_DUE_TO = " due to: {}";
protected static final String SUCCESSFULLY_EXECUTED_FUNCTION = SUCCESSFULLY_EXECUTED + FUNCTION;
protected static final String FAILED_TO_EXECUTE_FUNCTION_DUE_TO = FAILED_TO_EXECUTE + FUNCTION + FAILED_DUE_TO;
protected static final String SUCCESSFULLY_EXECUTED_DROP_STATEMENT = SUCCESSFULLY_EXECUTED + DROP_STATEMENT;
protected static final String FAILED_TO_EXECUTE_DROP_STATEMENT = FAILED_TO_EXECUTE + DROP_STATEMENT + FAILED_DUE_TO;
protected static final String SUCCESSFULLY_EXECUTED_QUERY = SUCCESSFULLY_EXECUTED + QUERY;
protected static final String FAILED_TO_EXECUTE_QUERY = FAILED_TO_EXECUTE + QUERY + FAILED_DUE_TO;
protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS ";
@Value("${spring.datasource.url}")
protected String dbUrl;
@ -67,7 +47,7 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
@Autowired
protected InstallScripts installScripts;
protected abstract void loadSql(Connection conn);
protected abstract void loadSql(Connection conn, String fileName);
protected void loadFunctions(Path sqlFile, Connection conn) throws Exception {
String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8);
@ -75,26 +55,46 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
}
protected boolean checkVersion(Connection conn) {
log.info("Check the current PostgreSQL version...");
boolean versionValid = false;
try {
CallableStatement callableStatement = conn.prepareCall("{? = " + CALL_CHECK_VERSION + " }");
callableStatement.registerOutParameter(1, Types.BOOLEAN);
callableStatement.execute();
versionValid = callableStatement.getBoolean(1);
callableStatement.close();
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT current_setting('server_version_num')");
resultSet.next();
if(resultSet.getLong(1) > 110000) {
versionValid = true;
}
statement.close();
} catch (Exception e) {
log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage());
}
return versionValid;
}
protected void executeFunction(Connection conn, String query) {
log.info("{} ... ", query);
protected boolean isOldSchema(Connection conn, long fromVersion) {
boolean isOldSchema = true;
try {
CallableStatement callableStatement = conn.prepareCall("{" + query + "}");
callableStatement.execute();
SQLWarning warnings = callableStatement.getWarnings();
Statement statement = conn.createStatement();
statement.execute("CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version));");
Thread.sleep(1000);
ResultSet resultSet = statement.executeQuery("SELECT schema_version FROM tb_schema_settings;");
if (resultSet.next()) {
isOldSchema = resultSet.getLong(1) <= fromVersion;
} else {
resultSet.close();
statement.execute("INSERT INTO tb_schema_settings (schema_version) VALUES (" + fromVersion + ")");
}
statement.close();
} catch (InterruptedException | SQLException e) {
log.info("Failed to check current PostgreSQL schema due to: {}", e.getMessage());
}
return isOldSchema;
}
protected void executeQuery(Connection conn, String query) {
try {
Statement statement = conn.createStatement();
statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
SQLWarning warnings = statement.getWarnings();
if (warnings != null) {
log.info("{}", warnings.getMessage());
SQLWarning nextWarning = warnings.getNextWarning();
@ -103,31 +103,10 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
nextWarning = nextWarning.getNextWarning();
}
}
callableStatement.close();
log.info(SUCCESSFULLY_EXECUTED_FUNCTION, query.replace(CALL_REGEX, ""));
Thread.sleep(2000);
} catch (Exception e) {
log.info(FAILED_TO_EXECUTE_FUNCTION_DUE_TO, query, e.getMessage());
}
}
protected void executeDropStatement(Connection conn, String query) {
try {
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
log.info(SUCCESSFULLY_EXECUTED_DROP_STATEMENT, query);
Thread.sleep(5000);
} catch (InterruptedException | SQLException e) {
log.info(FAILED_TO_EXECUTE_DROP_STATEMENT, query, e.getMessage());
}
}
protected void executeQuery(Connection conn, String query) {
try {
conn.createStatement().execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
log.info(SUCCESSFULLY_EXECUTED_QUERY, query);
Thread.sleep(5000);
log.info("Successfully executed query: {}", query);
} catch (InterruptedException | SQLException e) {
log.info(FAILED_TO_EXECUTE_QUERY, query, e.getMessage());
log.info("Failed to execute query: {} due to: {}", query, e.getMessage());
}
}

87
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java

@ -34,6 +34,8 @@ import java.sql.DriverManager;
public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql";
private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql";
private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql";
private static final String TS_KV_OLD = "ts_kv_old;";
private static final String TS_KV_LATEST_OLD = "ts_kv_latest_old;";
@ -57,52 +59,58 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD;
private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD;
private static final String DROP_FUNCTION_CHECK_VERSION = DROP_FUNCTION_IF_EXISTS + CHECK_VERSION;
private static final String DROP_FUNCTION_CREATE_PARTITION_TS_KV_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_PARTITION_TS_KV_TABLE;
private static final String DROP_FUNCTION_CREATE_NEW_TS_KV_LATEST_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_NEW_TS_KV_LATEST_TABLE;
private static final String DROP_FUNCTION_CREATE_PARTITIONS = DROP_FUNCTION_IF_EXISTS + CREATE_PARTITIONS;
private static final String DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String DROP_FUNCTION_INSERT_INTO_DICTIONARY = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_FUNCTION_INSERT_INTO_TS_KV = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_PARTITION_TS_KV_TABLE;
private static final String DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_LATEST_TABLE;
private static final String DROP_PROCEDURE_CREATE_PARTITIONS = DROP_PROCEDURE_IF_EXISTS + CREATE_PARTITIONS;
private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
@Override
public void upgradeDatabase(String fromVersion) throws Exception {
switch (fromVersion) {
case "2.4.3":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating timeseries schema ...");
log.info("Load upgrade functions ...");
loadSql(conn);
log.info("Check the current PostgreSQL version...");
boolean versionValid = checkVersion(conn);
if (!versionValid) {
log.info("PostgreSQL version should be at least more than 10!");
log.info("Please upgrade your PostgreSQL and restart the script!");
throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!");
} else {
log.info("PostgreSQL version is valid!");
log.info("Updating schema ...");
executeFunction(conn, CALL_CREATE_PARTITION_TS_KV_TABLE);
executeFunction(conn, CALL_CREATE_PARTITIONS);
executeFunction(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeFunction(conn, CALL_INSERT_INTO_DICTIONARY);
executeFunction(conn, CALL_INSERT_INTO_TS_KV);
executeFunction(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeFunction(conn, CALL_INSERT_INTO_TS_KV_LATEST);
executeDropStatement(conn, DROP_TABLE_TS_KV_OLD);
executeDropStatement(conn, DROP_TABLE_TS_KV_LATEST_OLD);
executeDropStatement(conn, DROP_FUNCTION_CHECK_VERSION);
executeDropStatement(conn, DROP_FUNCTION_CREATE_PARTITION_TS_KV_TABLE);
executeDropStatement(conn, DROP_FUNCTION_CREATE_PARTITIONS);
executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE);
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_DICTIONARY);
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV);
executeDropStatement(conn, DROP_FUNCTION_CREATE_NEW_TS_KV_LATEST_TABLE);
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN json_v json;");
executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;");
if (isOldSchema(conn, 2004003)) {
log.info("Load upgrade functions ...");
loadSql(conn, LOAD_FUNCTIONS_SQL);
log.info("Updating timeseries schema ...");
executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE);
executeQuery(conn, CALL_CREATE_PARTITIONS);
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
executeQuery(conn, CALL_INSERT_INTO_TS_KV);
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, DROP_TABLE_TS_KV_OLD);
executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD);
executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITION_TS_KV_TABLE);
executeQuery(conn, DROP_PROCEDURE_CREATE_PARTITIONS);
executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV);
executeQuery(conn, DROP_PROCEDURE_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");
executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;");
}
log.info("Load TTL functions ...");
loadSql(conn, LOAD_TTL_FUNCTIONS_SQL);
log.info("Load Drop Partitions functions ...");
loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL);
executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000");
log.info("schema timeseries updated!");
}
@ -113,11 +121,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
}
}
protected void loadSql(Connection conn) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL);
@Override
protected void loadSql(Connection conn, String fileName) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName);
try {
loadFunctions(schemaUpdateFile, conn);
log.info("Upgrade functions successfully loaded!");
log.info("Functions successfully loaded!");
} catch (Exception e) {
log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage());
}

4
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java

@ -45,13 +45,13 @@ public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaS
private long chunkTimeInterval;
public TimescaleTsDatabaseSchemaService() {
super("schema-timescale.sql", "schema-timescale-idx.sql");
super("schema-timescale.sql", null);
}
@Override
public void createDatabaseSchema() throws Exception {
super.createDatabaseSchema();
executeQuery("SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
executeQuery("SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
}
private void executeQuery(String query) {

95
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java

@ -39,31 +39,32 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private long chunkTimeInterval;
private static final String LOAD_FUNCTIONS_SQL = "schema_update_timescale_ts.sql";
private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql";
private static final String TENANT_TS_KV_OLD_TABLE = "tenant_ts_kv_old;";
private static final String CREATE_TS_KV_LATEST_TABLE = "create_ts_kv_latest_table()";
private static final String CREATE_NEW_TENANT_TS_KV_TABLE = "create_new_tenant_ts_kv_table()";
private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()";
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
private static final String INSERT_INTO_TENANT_TS_KV = "insert_into_tenant_ts_kv()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()";
private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE;
private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TENANT_TS_KV_TABLE;
private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE;
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TENANT_TS_KV;
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE;
private static final String DROP_FUNCTION_CREATE_TS_KV_LATEST_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_LATEST_TABLE;
private static final String DROP_FUNCTION_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_FUNCTION_IF_EXISTS + CREATE_NEW_TENANT_TS_KV_TABLE;
private static final String DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE = DROP_FUNCTION_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String DROP_FUNCTION_INSERT_INTO_DICTIONARY = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_FUNCTION_INSERT_INTO_TENANT_TS_KV = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TENANT_TS_KV;
private static final String DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST = DROP_FUNCTION_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_LATEST_TABLE;
private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE;
private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
@Autowired
private InstallScripts installScripts;
@ -73,41 +74,44 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
switch (fromVersion) {
case "2.4.3":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating timescale schema ...");
log.info("Load upgrade functions ...");
loadSql(conn);
log.info("Check the current PostgreSQL version...");
boolean versionValid = checkVersion(conn);
if (!versionValid) {
log.info("PostgreSQL version should be at least more than 9.6!");
log.info("Please upgrade your PostgreSQL and restart the script!");
throw new RuntimeException("PostgreSQL version should be at least more than 11, please upgrade your PostgreSQL and restart the script!");
} else {
log.info("PostgreSQL version is valid!");
log.info("Updating schema ...");
executeFunction(conn, CALL_CREATE_TS_KV_LATEST_TABLE);
executeFunction(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE);
executeQuery(conn, "SELECT create_hypertable('tenant_ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
executeFunction(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeFunction(conn, CALL_INSERT_INTO_DICTIONARY);
executeFunction(conn, CALL_INSERT_INTO_TS_KV);
executeFunction(conn, CALL_INSERT_INTO_TS_KV_LATEST);
//executeQuery(conn, "SELECT set_chunk_time_interval('tenant_ts_kv', " + chunkTimeInterval +");");
executeDropStatement(conn, DROP_OLD_TENANT_TS_KV_TABLE);
executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_LATEST_TABLE);
executeDropStatement(conn, DROP_FUNCTION_CREATE_TENANT_TS_KV_TABLE_COPY);
executeDropStatement(conn, DROP_FUNCTION_CREATE_TS_KV_DICTIONARY_TABLE);
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_DICTIONARY);
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TENANT_TS_KV);
executeDropStatement(conn, DROP_FUNCTION_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, "ALTER TABLE tenant_ts_kv ADD COLUMN json_v json;");
executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN json_v json;");
log.info("schema timeseries updated!");
if (isOldSchema(conn, 2004003)) {
log.info("Load upgrade functions ...");
loadSql(conn, LOAD_FUNCTIONS_SQL);
log.info("Updating timescale schema ...");
executeQuery(conn, CALL_CREATE_TS_KV_LATEST_TABLE);
executeQuery(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE);
executeQuery(conn, "SELECT create_hypertable('ts_kv', 'ts', chunk_time_interval => " + chunkTimeInterval + ", if_not_exists => true);");
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
executeQuery(conn, CALL_INSERT_INTO_TS_KV);
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE);
executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_LATEST_TABLE);
executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY);
executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");
executeQuery(conn, "ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS json_v json;");
}
log.info("Load TTL functions ...");
loadSql(conn, LOAD_TTL_FUNCTIONS_SQL);
executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000");
log.info("schema timescale updated!");
}
}
break;
@ -116,13 +120,14 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
}
}
protected void loadSql(Connection conn) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", LOAD_FUNCTIONS_SQL);
@Override
protected void loadSql(Connection conn, String fileName) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName);
try {
loadFunctions(schemaUpdateFile, conn);
log.info("Upgrade functions successfully loaded!");
log.info("Functions successfully loaded!");
} catch (Exception e) {
log.info("Failed to load Timescale upgrade functions due to: {}", e.getMessage());
log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage());
}
}
}

8
application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.script;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import delight.nashornsandbox.NashornSandbox;
import delight.nashornsandbox.NashornSandboxes;
import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
@ -28,20 +29,17 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.script.Invocable;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
@ -140,7 +138,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
if (maxRequestsTimeout > 0) {
result = Futures.withTimeout(result, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
Futures.addCallback(result, evalCallback);
Futures.addCallback(result, evalCallback, MoreExecutors.directExecutor());
return result;
}
@ -163,7 +161,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
if (maxRequestsTimeout > 0) {
result = Futures.withTimeout(result, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
Futures.addCallback(result, invokeCallback);
Futures.addCallback(result, invokeCallback, MoreExecutors.directExecutor());
return result;
}

9
application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.script;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
@ -158,7 +159,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
}
kafkaFailedMsgs.incrementAndGet();
}
});
}, MoreExecutors.directExecutor());
return Futures.transform(future, response -> {
JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse();
UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB());
@ -170,7 +171,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
log.debug("[{}] Failed to compile script due to [{}]: {}", compiledScriptId, compilationResult.getErrorCode().name(), compilationResult.getErrorDetails());
throw new RuntimeException(compilationResult.getErrorDetails());
}
});
}, MoreExecutors.directExecutor());
}
@Override
@ -209,7 +210,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
}
kafkaFailedMsgs.incrementAndGet();
}
});
}, MoreExecutors.directExecutor());
return Futures.transform(future, response -> {
JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse();
if (invokeResult.getSuccess()) {
@ -218,7 +219,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
log.debug("[{}] Failed to compile script due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails());
throw new RuntimeException(invokeResult.getErrorDetails());
}
});
}, MoreExecutors.directExecutor());
}
@Override

7
application/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java

@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.thingsboard.server.common.data.id.EntityId;
@ -121,7 +122,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
} else {
return Futures.immediateFuture(unbindMsg(json, msg));
}
});
}, MoreExecutors.directExecutor());
}
@Override
@ -174,7 +175,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
} else {
return Futures.immediateFuture(json.asBoolean());
}
});
}, MoreExecutors.directExecutor());
}
@Override
@ -232,7 +233,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
return Futures.immediateFailedFuture(new ScriptException(e));
}
}
});
}, MoreExecutors.directExecutor());
}
public void destroy() {

8
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -272,7 +272,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
log.warn("Failed to register device to the state service", t);
callback.onFailure(t);
}
});
}, MoreExecutors.directExecutor());
} else if (proto.getUpdated()) {
DeviceStateData stateData = getOrFetchDeviceStateData(device.getId());
if (stateData != null) {
@ -347,7 +347,7 @@ public class DefaultDeviceStateService implements DeviceStateService {
}
return null;
}
});
}, MoreExecutors.directExecutor());
fetchFutures.add(future);
}
}
@ -440,10 +440,10 @@ public class DefaultDeviceStateService implements DeviceStateService {
private ListenableFuture<DeviceStateData> fetchDeviceState(Device device) {
if (persistToTelemetry) {
ListenableFuture<List<TsKvEntry>> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES);
return Futures.transform(tsData, extractDeviceStateData(device));
return Futures.transform(tsData, extractDeviceStateData(device), MoreExecutors.directExecutor());
} else {
ListenableFuture<List<AttributeKvEntry>> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), DataConstants.SERVER_SCOPE, PERSISTENT_ATTRIBUTES);
return Futures.transform(attrData, extractDeviceStateData(device));
return Futures.transform(attrData, extractDeviceStateData(device), MoreExecutors.directExecutor());
}
}

14
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java

@ -21,6 +21,7 @@ import com.google.common.base.Function;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -76,7 +77,10 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -654,7 +658,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
}
ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
Futures.addCallback(future, callback);
Futures.addCallback(future, callback, MoreExecutors.directExecutor());
}
@Override
@ -668,7 +672,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
return new FutureCallback<ValidationResult>() {
@Override
public void onSuccess(@Nullable ValidationResult result) {
Futures.addCallback(attributesService.find(tenantId, entityId, scope, keys), callback);
Futures.addCallback(attributesService.find(tenantId, entityId, scope, keys), callback, MoreExecutors.directExecutor());
}
@Override
@ -688,7 +692,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
}
ListenableFuture<List<AttributeKvEntry>> future = mergeAllAttributesFutures(futures);
Futures.addCallback(future, callback);
Futures.addCallback(future, callback, MoreExecutors.directExecutor());
}
@Override
@ -702,7 +706,7 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi
return new FutureCallback<ValidationResult>() {
@Override
public void onSuccess(@Nullable ValidationResult result) {
Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), callback);
Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), callback, MoreExecutors.directExecutor());
}
@Override

11
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -80,14 +81,14 @@ public class DefaultTransportApiService implements TransportApiService {
TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue();
if (transportApiRequestMsg.hasValidateTokenRequestMsg()) {
ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg();
return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
} else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) {
ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg();
return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
} else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) {
return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
}
return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()));
return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
}
private ListenableFuture<TransportApiResponseMsg> validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) {
@ -147,7 +148,7 @@ public class DefaultTransportApiService implements TransportApiService {
log.warn("[{}] Failed to lookup device by id", deviceId, e);
return getEmptyTransportApiResponse();
}
});
}, MoreExecutors.directExecutor());
}
private DeviceInfoProto getDeviceInfoProto(Device device) throws JsonProcessingException {

89
application/src/main/java/org/thingsboard/server/service/ttl/AbstractTimeseriesCleanUpService.java

@ -0,0 +1,89 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.server.dao.util.PsqlTsAnyDao;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.sql.Statement;
@PsqlTsAnyDao
@Slf4j
public abstract class AbstractTimeseriesCleanUpService {
@Value("${sql.ttl.ts_key_value_ttl}")
protected long systemTtl;
@Value("${sql.ttl.enabled}")
private boolean ttlTaskExecutionEnabled;
@Value("${spring.datasource.url}")
private String dbUrl;
@Value("${spring.datasource.username}")
private String dbUserName;
@Value("${spring.datasource.password}")
private String dbPassword;
@Scheduled(initialDelayString = "${sql.ttl.execution_interval_ms}", fixedDelayString = "${sql.ttl.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
doCleanUp(conn);
} catch (SQLException e) {
log.error("SQLException occurred during TTL task execution ", e);
}
}
}
protected abstract void doCleanUp(Connection connection);
protected long executeQuery(Connection conn, String query) {
long removed = 0L;
try {
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery(query);
getWarnings(statement);
resultSet.next();
removed = resultSet.getLong(1);
log.debug("Successfully executed query: {}", query);
} catch (SQLException e) {
log.debug("Failed to execute query: {} due to: {}", query, e.getMessage());
}
return removed;
}
private void getWarnings(Statement statement) throws SQLException {
SQLWarning warnings = statement.getWarnings();
if (warnings != null) {
log.debug("{}", warnings.getMessage());
SQLWarning nextWarning = warnings.getNextWarning();
while (nextWarning != null) {
log.debug("{}", nextWarning.getMessage());
nextWarning = nextWarning.getNextWarning();
}
}
}
}

41
application/src/main/java/org/thingsboard/server/service/ttl/PsqlTimeseriesCleanUpService.java

@ -0,0 +1,41 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.util.PsqlTsDao;
import java.sql.Connection;
@PsqlTsDao
@Service
@Slf4j
public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
@Value("${sql.postgres.ts_key_value_partitioning}")
private String partitionType;
@Override
protected void doCleanUp(Connection connection) {
long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);");
log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved);
long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);");
log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
}
}

35
application/src/main/java/org/thingsboard/server/service/ttl/TimescaleTimeseriesCleanUpService.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.ttl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.sql.Connection;
@TimescaleDBTsDao
@Service
@Slf4j
public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
@Override
protected void doCleanUp(Connection connection) {
long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);");
log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
}
}

4
application/src/main/resources/thingsboard.yml

@ -202,6 +202,10 @@ sql:
timescale:
# Specify Interval size for new data chunks storage.
chunk_time_interval: "${SQL_TIMESCALE_CHUNK_TIME_INTERVAL:604800000}"
ttl:
enabled: "${SQL_TTL_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_EXECUTION_INTERVAL:86400000}" # Number of miliseconds
ts_key_value_ttl: "${SQL_TTL_TS_KEY_VALUE_TTL:0}" # Number of seconds
# Actor system parameters
actors:

4
application/src/main/scripts/control/deb/postinst

@ -2,8 +2,8 @@
set -e
chown -R ${pkg.name}: ${pkg.logFolder}
chown -R ${pkg.name}: ${pkg.installFolder}
chown -R ${pkg.user}: ${pkg.logFolder}
chown -R ${pkg.user}: ${pkg.installFolder}
systemctl --no-reload enable ${pkg.name}.service >/dev/null 2>&1 || :
exit 0

10
application/src/main/scripts/control/deb/preinst

@ -2,21 +2,21 @@
set -e
if ! getent group ${pkg.name} >/dev/null; then
addgroup --system ${pkg.name}
if ! getent group ${pkg.user} >/dev/null; then
addgroup --system ${pkg.user}
fi
if ! getent passwd ${pkg.name} >/dev/null; then
if ! getent passwd ${pkg.user} >/dev/null; then
adduser --quiet \
--system \
--ingroup ${pkg.name} \
--ingroup ${pkg.user} \
--quiet \
--disabled-login \
--disabled-password \
--home ${pkg.installFolder} \
--no-create-home \
-gecos "Thingsboard application" \
${pkg.name}
${pkg.user}
fi
exit 0

4
application/src/main/scripts/control/rpm/postinst

@ -1,7 +1,7 @@
#!/bin/sh
chown -R ${pkg.name}: ${pkg.logFolder}
chown -R ${pkg.name}: ${pkg.installFolder}
chown -R ${pkg.user}: ${pkg.logFolder}
chown -R ${pkg.user}: ${pkg.installFolder}
if [ $1 -eq 1 ] ; then
# Initial installation

6
application/src/main/scripts/control/rpm/preinst

@ -1,6 +1,6 @@
#!/bin/sh
getent group ${pkg.name} >/dev/null || groupadd -r ${pkg.name}
getent passwd ${pkg.name} >/dev/null || \
useradd -d ${pkg.installFolder} -g ${pkg.name} -M -r ${pkg.name} -s /sbin/nologin \
getent group ${pkg.user} >/dev/null || groupadd -r ${pkg.user}
getent passwd ${pkg.user} >/dev/null || \
useradd -d ${pkg.installFolder} -g ${pkg.user} -M -r ${pkg.user} -s /sbin/nologin \
-c "Thingsboard application"

2
application/src/main/scripts/control/thingsboard.service

@ -3,7 +3,7 @@ Description=${pkg.name}
After=syslog.target
[Service]
User=${pkg.name}
User=${pkg.user}
ExecStart=${pkg.installFolder}/bin/${pkg.name}.jar
SuccessExitStatus=143

2
application/src/main/scripts/install/install.sh

@ -44,7 +44,7 @@ installDir=${pkg.installFolder}/data
source "${CONF_FOLDER}/${configfile}"
run_user=${pkg.name}
run_user=${pkg.user}
su -s /bin/sh -c "java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \
-Dinstall.data_dir=${installDir} \

2
application/src/main/scripts/install/upgrade.sh

@ -43,7 +43,7 @@ installDir=${pkg.installFolder}/data
source "${CONF_FOLDER}/${configfile}"
run_user=${pkg.name}
run_user=${pkg.user}
su -s /bin/sh -c "java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \
-Dinstall.data_dir=${installDir} \

2
common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java

@ -39,7 +39,7 @@ public interface AlarmService {
ListenableFuture<Boolean> ackAlarm(TenantId tenantId, AlarmId alarmId, long ackTs);
ListenableFuture<Boolean> clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long ackTs);
ListenableFuture<Boolean> clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long clearTs);
ListenableFuture<Alarm> findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId);

21
common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
@ConditionalOnExpression("'${database.ts.type}'=='sql' && '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'")
public @interface PsqlTsDao { }

3
common/queue/src/main/java/org/thingsboard/server/queue/common/AsyncCallbackTemplate.java

@ -18,6 +18,7 @@ package org.thingsboard.server.queue.common;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
@ -59,7 +60,7 @@ public class AsyncCallbackTemplate {
if (executor != null) {
Futures.addCallback(future, callback, executor);
} else {
Futures.addCallback(future, callback);
Futures.addCallback(future, callback, MoreExecutors.directExecutor());
}
}

2
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java

@ -97,7 +97,7 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
callback.onFailure(t);
}
}
});
}, producerExecutor);
}
@Override

11
common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java

@ -18,19 +18,20 @@ package org.thingsboard.common.util;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.concurrent.Executor;
import java.util.function.Consumer;
public class DonAsynchron {
public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
Consumer<Throwable> onFailure) {
public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
Consumer<Throwable> onFailure) {
withCallback(future, onSuccess, onFailure, null);
}
public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
Consumer<Throwable> onFailure, Executor executor) {
public static <T> void withCallback(ListenableFuture<T> future, Consumer<T> onSuccess,
Consumer<Throwable> onFailure, Executor executor) {
FutureCallback<T> callback = new FutureCallback<T>() {
@Override
public void onSuccess(T result) {
@ -49,7 +50,7 @@ public class DonAsynchron {
if (executor != null) {
Futures.addCallback(future, callback, executor);
} else {
Futures.addCallback(future, callback);
Futures.addCallback(future, callback, MoreExecutors.directExecutor());
}
}
}

13
dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java

@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -53,7 +54,6 @@ import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
@ -264,9 +264,8 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
entityService.fetchEntityNameAsync(tenantId, alarmInfo.getOriginator()), originatorName -> {
alarmInfo.setOriginatorName(originatorName);
return alarmInfo;
}
);
});
}, MoreExecutors.directExecutor());
}, MoreExecutors.directExecutor());
}
@Override
@ -283,11 +282,11 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
}
alarmInfo.setOriginatorName(originatorName);
return alarmInfo;
}
}, MoreExecutors.directExecutor()
));
}
return Futures.successfulAsList(alarmFutures);
});
}, MoreExecutors.directExecutor());
}
return Futures.transform(alarms, new Function<List<AlarmInfo>, TimePageData<AlarmInfo>>() {
@Nullable
@ -295,7 +294,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
public TimePageData<AlarmInfo> apply(@Nullable List<AlarmInfo> alarms) {
return new TimePageData<>(alarms, query.getPageLink());
}
});
}, MoreExecutors.directExecutor());
}
@Override

13
dao/src/main/java/org/thingsboard/server/dao/alarm/CassandraAlarmDao.java

@ -20,6 +20,7 @@ import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.datastax.driver.core.querybuilder.Select;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@ -82,10 +83,10 @@ public class CassandraAlarmDao extends CassandraAbstractModelDao<AlarmEntity, Al
@Override
public Boolean deleteAlarm(TenantId tenantId, Alarm alarm) {
Statement delete = QueryBuilder.delete().all().from(getColumnFamilyName()).where(eq(ModelConstants.ID_PROPERTY, alarm.getId().getId()))
.and(eq(ALARM_TENANT_ID_PROPERTY, tenantId.getId()))
.and(eq(ALARM_ORIGINATOR_ID_PROPERTY, alarm.getOriginator().getId()))
.and(eq(ALARM_ORIGINATOR_TYPE_PROPERTY, alarm.getOriginator().getEntityType()))
.and(eq(ALARM_TYPE_PROPERTY, alarm.getType()));
.and(eq(ALARM_TENANT_ID_PROPERTY, tenantId.getId()))
.and(eq(ALARM_ORIGINATOR_ID_PROPERTY, alarm.getOriginator().getId()))
.and(eq(ALARM_ORIGINATOR_TYPE_PROPERTY, alarm.getOriginator().getEntityType()))
.and(eq(ALARM_TYPE_PROPERTY, alarm.getType()));
log.debug("Remove request: {}", delete.toString());
return executeWrite(tenantId, delete).wasApplied();
}
@ -122,10 +123,10 @@ public class CassandraAlarmDao extends CassandraAbstractModelDao<AlarmEntity, Al
for (EntityRelation relation : input) {
alarmFutures.add(Futures.transform(
findAlarmByIdAsync(tenantId, relation.getTo().getId()),
AlarmInfo::new));
AlarmInfo::new, MoreExecutors.directExecutor()));
}
return Futures.successfulAsList(alarmFutures);
});
}, MoreExecutors.directExecutor());
}
@Override

34
dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java

@ -16,9 +16,9 @@
package org.thingsboard.server.dao.asset;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.beans.factory.annotation.Autowired;
@ -29,7 +29,6 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntitySubtype;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
@ -62,7 +61,10 @@ import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.CacheConstants.ASSET_CACHE;
import static org.thingsboard.server.dao.DaoUtil.toUUIDs;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
import static org.thingsboard.server.dao.service.Validator.*;
import static org.thingsboard.server.dao.service.Validator.validateId;
import static org.thingsboard.server.dao.service.Validator.validateIds;
import static org.thingsboard.server.dao.service.Validator.validatePageLink;
import static org.thingsboard.server.dao.service.Validator.validateString;
@Service
@Slf4j
@ -258,9 +260,9 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ
}
}
return Futures.successfulAsList(futures);
});
}, MoreExecutors.directExecutor());
assets = Futures.transform(assets, assetList ->
assetList == null ? Collections.emptyList() : assetList.stream().filter(asset -> query.getAssetTypes().contains(asset.getType())).collect(Collectors.toList())
assetList == null ? Collections.emptyList() : assetList.stream().filter(asset -> query.getAssetTypes().contains(asset.getType())).collect(Collectors.toList()), MoreExecutors.directExecutor()
);
return assets;
}
@ -274,7 +276,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ
assetTypes -> {
assetTypes.sort(Comparator.comparing(EntitySubtype::getType));
return assetTypes;
});
}, MoreExecutors.directExecutor());
}
private DataValidator<Asset> assetValidator =
@ -335,18 +337,18 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ
};
private PaginatedRemover<TenantId, Asset> tenantAssetsRemover =
new PaginatedRemover<TenantId, Asset>() {
new PaginatedRemover<TenantId, Asset>() {
@Override
protected List<Asset> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
return assetDao.findAssetsByTenantId(id.getId(), pageLink);
}
@Override
protected List<Asset> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
return assetDao.findAssetsByTenantId(id.getId(), pageLink);
}
@Override
protected void removeEntity(TenantId tenantId, Asset entity) {
deleteAsset(tenantId, new AssetId(entity.getId().getId()));
}
};
@Override
protected void removeEntity(TenantId tenantId, Asset entity) {
deleteAsset(tenantId, new AssetId(entity.getId().getId()));
}
};
private PaginatedRemover<CustomerId, Asset> customerAssetsUnasigner = new PaginatedRemover<CustomerId, Asset>() {

3
dao/src/main/java/org/thingsboard/server/dao/asset/CassandraAssetDao.java

@ -23,6 +23,7 @@ import com.datastax.driver.mapping.Result;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.EntitySubtype;
@ -185,7 +186,7 @@ public class CassandraAssetDao extends CassandraAbstractSearchTextDao<AssetEntit
return Collections.emptyList();
}
}
});
}, MoreExecutors.directExecutor());
}
}

3
dao/src/main/java/org/thingsboard/server/dao/dashboard/CassandraDashboardInfoDao.java

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.dashboard;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@ -85,7 +86,7 @@ public class CassandraDashboardInfoDao extends CassandraAbstractSearchTextDao<Da
dashboardFutures.add(findByIdAsync(new TenantId(tenantId), relation.getTo().getId()));
}
return Futures.successfulAsList(dashboardFutures);
});
}, MoreExecutors.directExecutor());
}
}

37
dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.dashboard;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -64,7 +65,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
@Autowired
private TenantDao tenantDao;
@Autowired
private CustomerDao customerDao;
@ -102,7 +103,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
dashboardValidator.validate(dashboard, DashboardInfo::getTenantId);
return dashboardDao.save(dashboard.getTenantId(), dashboard);
}
@Override
public Dashboard assignDashboardToCustomer(TenantId tenantId, DashboardId dashboardId, CustomerId customerId) {
Dashboard dashboard = findDashboardById(tenantId, dashboardId);
@ -203,7 +204,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
public TimePageData<DashboardInfo> apply(@Nullable List<DashboardInfo> dashboards) {
return new TimePageData<>(dashboards, pageLink);
}
});
}, MoreExecutors.directExecutor());
}
@Override
@ -244,24 +245,24 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
}
}
}
};
};
private PaginatedRemover<TenantId, DashboardInfo> tenantDashboardsRemover =
new PaginatedRemover<TenantId, DashboardInfo>() {
@Override
protected List<DashboardInfo> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink);
}
@Override
protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
deleteDashboard(tenantId, new DashboardId(entity.getUuidId()));
}
};
@Override
protected List<DashboardInfo> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink);
}
@Override
protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
deleteDashboard(tenantId, new DashboardId(entity.getUuidId()));
}
};
private class CustomerDashboardsUnassigner extends TimePaginatedRemover<Customer, DashboardInfo> {
private Customer customer;
CustomerDashboardsUnassigner(Customer customer) {
@ -282,7 +283,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
protected void removeEntity(TenantId tenantId, DashboardInfo entity) {
unassignDashboardFromCustomer(customer.getTenantId(), new DashboardId(entity.getUuidId()), this.customer.getId());
}
}
private class CustomerDashboardsUpdater extends TimePaginatedRemover<Customer, DashboardInfo> {

5
dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java

@ -23,6 +23,7 @@ import com.datastax.driver.mapping.Result;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.Device;
@ -178,14 +179,14 @@ public class CassandraDeviceDao extends CassandraAbstractSearchTextDao<DeviceEnt
if (result != null) {
List<EntitySubtype> entitySubtypes = new ArrayList<>();
result.all().forEach((entitySubtypeEntity) ->
entitySubtypes.add(entitySubtypeEntity.toEntitySubtype())
entitySubtypes.add(entitySubtypeEntity.toEntitySubtype())
);
return entitySubtypes;
} else {
return Collections.emptyList();
}
}
});
}, MoreExecutors.directExecutor());
}
}

9
dao/src/main/java/org/thingsboard/server/dao/device/ClaimDevicesServiceImpl.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -97,9 +98,9 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService {
}
log.warn("Failed to find claimingAllowed attribute for device or it is already claimed![{}]", device.getName());
throw new IllegalArgumentException();
});
}, MoreExecutors.directExecutor());
}
});
}, MoreExecutors.directExecutor());
}
private ClaimDataInfo getClaimData(Cache cache, Device device) throws ExecutionException, InterruptedException {
@ -138,9 +139,9 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService {
if (device.getCustomerId().getId().equals(ModelConstants.NULL_UUID)) {
device.setCustomerId(customerId);
Device savedDevice = deviceService.saveDevice(device);
return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(savedDevice, ClaimResponse.SUCCESS));
return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(savedDevice, ClaimResponse.SUCCESS), MoreExecutors.directExecutor());
}
return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(null, ClaimResponse.CLAIMED));
return Futures.transform(removeClaimingSavedData(cache, claimData, device), result -> new ClaimResult(null, ClaimResponse.CLAIMED), MoreExecutors.directExecutor());
}
} else {
log.warn("Failed to find the device's claiming message![{}]", device.getName());

27
dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.hibernate.exception.ConstraintViolationException;
@ -291,7 +292,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
}
}
return Futures.successfulAsList(futures);
});
}, MoreExecutors.directExecutor());
devices = Futures.transform(devices, new Function<List<Device>, List<Device>>() {
@Nullable
@ -299,7 +300,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
public List<Device> apply(@Nullable List<Device> deviceList) {
return deviceList == null ? Collections.emptyList() : deviceList.stream().filter(device -> query.getDeviceTypes().contains(device.getType())).collect(Collectors.toList());
}
});
}, MoreExecutors.directExecutor());
return devices;
}
@ -313,7 +314,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
deviceTypes -> {
deviceTypes.sort(Comparator.comparing(EntitySubtype::getType));
return deviceTypes;
});
}, MoreExecutors.directExecutor());
}
private DataValidator<Device> deviceValidator =
@ -374,18 +375,18 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
};
private PaginatedRemover<TenantId, Device> tenantDevicesRemover =
new PaginatedRemover<TenantId, Device>() {
new PaginatedRemover<TenantId, Device>() {
@Override
protected List<Device> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
return deviceDao.findDevicesByTenantId(id.getId(), pageLink);
}
@Override
protected List<Device> findEntities(TenantId tenantId, TenantId id, TextPageLink pageLink) {
return deviceDao.findDevicesByTenantId(id.getId(), pageLink);
}
@Override
protected void removeEntity(TenantId tenantId, Device entity) {
deleteDevice(tenantId, new DeviceId(entity.getUuidId()));
}
};
@Override
protected void removeEntity(TenantId tenantId, Device entity) {
deleteDevice(tenantId, new DeviceId(entity.getUuidId()));
}
};
private PaginatedRemover<CustomerId, Device> customerDeviceUnasigner = new PaginatedRemover<CustomerId, Device>() {

5
dao/src/main/java/org/thingsboard/server/dao/entityview/CassandraEntityViewDao.java

@ -23,6 +23,7 @@ import com.datastax.driver.mapping.Result;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.EntitySubtype;
@ -97,7 +98,7 @@ public class CassandraEntityViewDao extends CassandraAbstractSearchTextDao<Entit
log.debug("Try to find entity views by tenantId [{}] and pageLink [{}]", tenantId, pageLink);
List<EntityViewEntity> entityViewEntities =
findPageWithTextSearch(new TenantId(tenantId), ENTITY_VIEW_BY_TENANT_AND_SEARCH_TEXT_CF,
Collections.singletonList(eq(TENANT_ID_PROPERTY, tenantId)), pageLink);
Collections.singletonList(eq(TENANT_ID_PROPERTY, tenantId)), pageLink);
log.trace("Found entity views [{}] by tenantId [{}] and pageLink [{}]",
entityViewEntities, tenantId, pageLink);
return DaoUtil.convertDataList(entityViewEntities);
@ -181,6 +182,6 @@ public class CassandraEntityViewDao extends CassandraAbstractSearchTextDao<Entit
return Collections.emptyList();
}
}
});
}, MoreExecutors.directExecutor());
}
}

9
dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java

@ -19,6 +19,7 @@ import com.google.common.base.Function;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -199,7 +200,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
}
}
return Futures.successfulAsList(futures);
});
}, MoreExecutors.directExecutor());
entityViews = Futures.transform(entityViews, new Function<List<EntityView>, List<EntityView>>() {
@Nullable
@ -207,7 +208,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
public List<EntityView> apply(@Nullable List<EntityView> entityViewList) {
return entityViewList == null ? Collections.emptyList() : entityViewList.stream().filter(entityView -> query.getEntityViewTypes().contains(entityView.getType())).collect(Collectors.toList());
}
});
}, MoreExecutors.directExecutor());
return entityViews;
}
@ -246,7 +247,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
public void onFailure(Throwable t) {
log.error("Error while finding entity views by tenantId and entityId", t);
}
});
}, MoreExecutors.directExecutor());
return entityViewsFuture;
}
}
@ -279,7 +280,7 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti
entityViewTypes -> {
entityViewTypes.sort(Comparator.comparing(EntitySubtype::getType));
return entityViewTypes;
});
}, MoreExecutors.directExecutor());
}
private DataValidator<EntityView> entityViewValidator =

11
dao/src/main/java/org/thingsboard/server/dao/event/CassandraBaseEventDao.java

@ -22,6 +22,7 @@ import com.datastax.driver.core.querybuilder.Select;
import com.datastax.driver.core.utils.UUIDs;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
@ -45,10 +46,12 @@ import java.util.UUID;
import java.util.concurrent.ExecutionException;
import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
import static com.datastax.driver.core.querybuilder.QueryBuilder.in;
import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
import static com.datastax.driver.core.querybuilder.QueryBuilder.ttl;
import static org.thingsboard.server.dao.model.ModelConstants.*;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BY_ID_VIEW_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BY_TYPE_AND_ID_VIEW_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@Component
@Slf4j
@ -96,7 +99,7 @@ public class CassandraBaseEventDao extends CassandraAbstractSearchTimeDao<EventE
event.setUid(event.getId().toString());
}
ListenableFuture<Optional<Event>> optionalSave = saveAsync(event.getTenantId(), new EventEntity(event), false, eventsTtl);
return Futures.transform(optionalSave, opt -> opt.orElse(null));
return Futures.transform(optionalSave, opt -> opt.orElse(null), MoreExecutors.directExecutor());
}
@Override
@ -210,6 +213,6 @@ public class CassandraBaseEventDao extends CassandraAbstractSearchTimeDao<EventE
} else {
return Optional.empty();
}
});
}, MoreExecutors.directExecutor());
}
}

5
dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java

@ -36,6 +36,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.BOOLEAN_VALUE_COLU
import static org.thingsboard.server.dao.model.ModelConstants.DOUBLE_VALUE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.JSON_VALUE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.LONG_VALUE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
@ -53,6 +54,10 @@ public abstract class AbstractTsKvEntity implements ToData<TsKvEntry> {
@Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid")
protected UUID entityId;
@Id
@Column(name = KEY_COLUMN)
protected int key;
@Id
@Column(name = TS_COLUMN)
protected Long ts;

4
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java

@ -69,10 +69,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
})
public final class TsKvLatestEntity extends AbstractTsKvEntity {
@Id
@Column(name = KEY_COLUMN)
private int key;
@Override
public boolean isNotEmpty() {
return strValue != null || longValue != null || doubleValue != null || booleanValue != null || jsonValue != null;

1
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvCompositeKey.java

@ -31,7 +31,6 @@ public class TimescaleTsKvCompositeKey implements Serializable {
@Transient
private static final long serialVersionUID = -4089175869616037523L;
private UUID tenantId;
private UUID entityId;
private int key;
private long ts;

19
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/timescale/ts/TimescaleTsKvEntity.java

@ -18,25 +18,18 @@ package org.thingsboard.server.dao.model.sqlts.timescale.ts;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.model.ToData;
import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
import javax.persistence.Column;
import javax.persistence.ColumnResult;
import javax.persistence.ConstructorResult;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.NamedNativeQueries;
import javax.persistence.NamedNativeQuery;
import javax.persistence.SqlResultSetMapping;
import javax.persistence.SqlResultSetMappings;
import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.TENANT_ID_COLUMN;
import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG;
import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_AVG_QUERY;
import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.FIND_COUNT;
@ -52,7 +45,7 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@Table(name = "tenant_ts_kv")
@Table(name = "ts_kv")
@IdClass(TimescaleTsKvCompositeKey.class)
@SqlResultSetMappings({
@SqlResultSetMapping(
@ -116,15 +109,7 @@ import static org.thingsboard.server.dao.sqlts.timescale.AggregationRepository.F
resultSetMapping = "timescaleCountMapping"
)
})
public final class TimescaleTsKvEntity extends AbstractTsKvEntity implements ToData<TsKvEntry> {
@Id
@Column(name = TENANT_ID_COLUMN, columnDefinition = "uuid")
private UUID tenantId;
@Id
@Column(name = KEY_COLUMN)
private int key;
public final class TimescaleTsKvEntity extends AbstractTsKvEntity {
public TimescaleTsKvEntity() {
}

4
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/ts/TsKvEntity.java

@ -32,10 +32,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN;
@IdClass(TsKvCompositeKey.class)
public final class TsKvEntity extends AbstractTsKvEntity {
@Id
@Column(name = KEY_COLUMN)
private int key;
public TsKvEntity() {
}

7
dao/src/main/java/org/thingsboard/server/dao/nosql/CassandraAbstractModelDao.java

@ -26,6 +26,7 @@ import com.datastax.driver.mapping.Result;
import com.google.common.base.Function;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.Dao;
@ -86,7 +87,7 @@ public abstract class CassandraAbstractModelDao<E extends BaseEntity<D>, D> exte
return Collections.emptyList();
}
}
});
}, MoreExecutors.directExecutor());
}
return Futures.immediateFuture(Collections.emptyList());
}
@ -120,7 +121,7 @@ public abstract class CassandraAbstractModelDao<E extends BaseEntity<D>, D> exte
return null;
}
}
});
}, MoreExecutors.directExecutor());
}
return Futures.immediateFuture(null);
}
@ -191,5 +192,5 @@ public abstract class CassandraAbstractModelDao<E extends BaseEntity<D>, D> exte
List<E> entities = findListByStatement(tenantId, QueryBuilder.select().all().from(getColumnFamilyName()).setConsistencyLevel(cluster.getDefaultReadConsistencyLevel()));
return DaoUtil.convertDataList(entities);
}
}

7
dao/src/main/java/org/thingsboard/server/dao/nosql/RateLimitedResultSetFuture.java

@ -22,6 +22,7 @@ import com.datastax.driver.core.Statement;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.Uninterruptibles;
import org.thingsboard.server.dao.exception.BufferLimitException;
import org.thingsboard.server.dao.util.AsyncRateLimiter;
@ -44,9 +45,9 @@ public class RateLimitedResultSetFuture implements ResultSetFuture {
rateLimiter.release();
}
return Futures.immediateFailedFuture(t);
});
}, MoreExecutors.directExecutor());
this.originalFuture = Futures.transform(rateLimitFuture,
i -> executeAsyncWithRelease(rateLimiter, session, statement));
i -> executeAsyncWithRelease(rateLimiter, session, statement), MoreExecutors.directExecutor());
}
@ -145,7 +146,7 @@ public class RateLimitedResultSetFuture implements ResultSetFuture {
public void onFailure(Throwable t) {
rateLimiter.release();
}
});
}, MoreExecutors.directExecutor());
return resultSetFuture;
} catch (RuntimeException re) {
rateLimiter.release();

5
dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.alarm;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
@ -108,9 +109,9 @@ public class JpaAlarmDao extends JpaAbstractDao<AlarmEntity, Alarm> implements A
for (EntityRelation relation : input) {
alarmFutures.add(Futures.transform(
findAlarmByIdAsync(tenantId, relation.getTo().getId()),
AlarmInfo::new));
AlarmInfo::new, MoreExecutors.directExecutor()));
}
return Futures.successfulAsList(alarmFutures);
});
}, MoreExecutors.directExecutor());
}
}

3
dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.dashboard;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
@ -91,6 +92,6 @@ public class JpaDashboardInfoDao extends JpaAbstractSearchTextDao<DashboardInfoE
dashboardFutures.add(findByIdAsync(new TenantId(tenantId), relation.getTo().getId()));
}
return Futures.successfulAsList(dashboardFutures);
});
}, MoreExecutors.directExecutor());
}
}

39
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sqlts;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@ -95,7 +96,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
@Override
public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
return getRemoveLatestFuture(tenantId, entityId, query);
return getRemoveLatestFuture(entityId, query);
}
@Override
@ -124,9 +125,9 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
}
@Override
protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
protected ListenableFuture<List<TsKvEntry>> findAllAsync(EntityId entityId, ReadTsKvQuery query) {
if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(tenantId, entityId, query);
return findAllAsyncWithLimit(entityId, query);
} else {
long stepTs = query.getStartTs();
List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
@ -134,7 +135,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
long startTs = stepTs;
long endTs = stepTs + query.getInterval();
long ts = startTs + (endTs - startTs) / 2;
futures.add(findAndAggregateAsync(tenantId, entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
stepTs = endTs;
}
return getTskvEntriesFuture(Futures.allAsList(futures));
@ -142,7 +143,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
}
@Override
protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) {
Integer keyId = getOrSaveKeyId(query.getKey());
List<TsKvEntity> tsKvEntities = tsKvRepository.findAllWithLimit(
entityId.getId(),
@ -156,9 +157,9 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities));
}
protected ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
switchAggregation(tenantId, entityId, key, startTs, endTs, aggregation, entitiesFutures);
switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures);
return Futures.transform(setFutures(entitiesFutures), entity -> {
if (entity != null && entity.isNotEmpty()) {
entity.setEntityId(entityId.getId());
@ -168,32 +169,32 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
} else {
return Optional.empty();
}
});
}, MoreExecutors.directExecutor());
}
protected void switchAggregation(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
protected void switchAggregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
switch (aggregation) {
case AVG:
findAvg(tenantId, entityId, key, startTs, endTs, entitiesFutures);
findAvg(entityId, key, startTs, endTs, entitiesFutures);
break;
case MAX:
findMax(tenantId, entityId, key, startTs, endTs, entitiesFutures);
findMax(entityId, key, startTs, endTs, entitiesFutures);
break;
case MIN:
findMin(tenantId, entityId, key, startTs, endTs, entitiesFutures);
findMin(entityId, key, startTs, endTs, entitiesFutures);
break;
case SUM:
findSum(tenantId, entityId, key, startTs, endTs, entitiesFutures);
findSum(entityId, key, startTs, endTs, entitiesFutures);
break;
case COUNT:
findCount(tenantId, entityId, key, startTs, endTs, entitiesFutures);
findCount(entityId, key, startTs, endTs, entitiesFutures);
break;
default:
throw new IllegalArgumentException("Not supported aggregation type: " + aggregation);
}
}
protected void findCount(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
protected void findCount(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
Integer keyId = getOrSaveKeyId(key);
entitiesFutures.add(tsKvRepository.findCount(
entityId.getId(),
@ -202,7 +203,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
endTs));
}
protected void findSum(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
protected void findSum(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
Integer keyId = getOrSaveKeyId(key);
entitiesFutures.add(tsKvRepository.findSum(
entityId.getId(),
@ -211,7 +212,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
endTs));
}
protected void findMin(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
protected void findMin(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
Integer keyId = getOrSaveKeyId(key);
entitiesFutures.add(tsKvRepository.findStringMin(
entityId.getId(),
@ -225,7 +226,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
endTs));
}
protected void findMax(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
protected void findMax(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
Integer keyId = getOrSaveKeyId(key);
entitiesFutures.add(tsKvRepository.findStringMax(
entityId.getId(),
@ -239,7 +240,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
endTs));
}
protected void findAvg(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
protected void findAvg(EntityId entityId, String key, long startTs, long endTs, List<CompletableFuture<TsKvEntity>> entitiesFutures) {
Integer keyId = getOrSaveKeyId(key);
entitiesFutures.add(tsKvRepository.findAvg(
entityId.getId(),

21
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java

@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.beans.factory.annotation.Autowired;
@ -126,7 +127,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
protected ListenableFuture<List<TsKvEntry>> processFindAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
List<ListenableFuture<List<TsKvEntry>>> futures = queries
.stream()
.map(query -> findAllAsync(tenantId, entityId, query))
.map(query -> findAllAsync(entityId, query))
.collect(Collectors.toList());
return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() {
@Nullable
@ -143,9 +144,9 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
}, service);
}
protected abstract ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query);
protected abstract ListenableFuture<List<TsKvEntry>> findAllAsync(EntityId entityId, ReadTsKvQuery query);
protected abstract ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query);
protected abstract ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query);
protected ListenableFuture<List<TsKvEntry>> getTskvEntriesFuture(ListenableFuture<List<Optional<TsKvEntry>>> future) {
return Futures.transform(future, new Function<List<Optional<TsKvEntry>>, List<TsKvEntry>>() {
@ -163,12 +164,12 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
}, service);
}
protected ListenableFuture<List<TsKvEntry>> findNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
protected ListenableFuture<List<TsKvEntry>> findNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
long startTs = 0;
long endTs = query.getStartTs() - 1;
ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1,
Aggregation.NONE, DESC_ORDER);
return findAllAsync(tenantId, entityId, findNewLatestQuery);
return findAllAsync(entityId, findNewLatestQuery);
}
protected ListenableFuture<TsKvEntry> getFindLatestFuture(EntityId entityId, String key) {
@ -188,7 +189,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
return Futures.immediateFuture(result);
}
protected ListenableFuture<Void> getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
protected ListenableFuture<Void> getRemoveLatestFuture(EntityId entityId, DeleteTsKvQuery query) {
ListenableFuture<TsKvEntry> latestFuture = getFindLatestFuture(entityId, query.getKey());
ListenableFuture<Boolean> booleanFuture = Futures.transform(latestFuture, tsKvEntry -> {
@ -216,7 +217,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
if (query.getRewriteLatestIfDeleted()) {
ListenableFuture<Void> savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> {
if (isRemove) {
return getNewLatestEntryFuture(tenantId, entityId, query);
return getNewLatestEntryFuture(entityId, query);
}
return Futures.immediateFuture(null);
}, service);
@ -235,7 +236,7 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
public void onFailure(Throwable t) {
log.warn("[{}] Failed to process remove of the latest value", entityId, t);
}
});
}, MoreExecutors.directExecutor());
return resultFuture;
}
@ -295,8 +296,8 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx
return keyId;
}
private ListenableFuture<Void> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
private ListenableFuture<Void> getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) {
ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(entityId, query);
return Futures.transformAsync(future, entryList -> {
if (entryList.size() == 1) {
return getSaveLatestFuture(entityId, entryList.get(0));

43
dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/timescale/TimescaleInsertTsRepository.java

@ -37,8 +37,8 @@ import java.util.List;
public class TimescaleInsertTsRepository extends AbstractInsertRepository implements InsertTsRepository<TimescaleTsKvEntity> {
private static final String INSERT_OR_UPDATE =
"INSERT INTO tenant_ts_kv (tenant_id, entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES(?, ?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " +
"ON CONFLICT (tenant_id, entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json);";
"INSERT INTO ts_kv (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " +
"ON CONFLICT (entity_id, key, ts) DO UPDATE SET bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json);";
@Override
public void saveOrUpdate(List<EntityContainer<TimescaleTsKvEntity>> entities) {
@ -46,41 +46,40 @@ public class TimescaleInsertTsRepository extends AbstractInsertRepository implem
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
TimescaleTsKvEntity tsKvEntity = entities.get(i).getEntity();
ps.setObject(1, tsKvEntity.getTenantId());
ps.setObject(2, tsKvEntity.getEntityId());
ps.setInt(3, tsKvEntity.getKey());
ps.setLong(4, tsKvEntity.getTs());
ps.setObject(1, tsKvEntity.getEntityId());
ps.setInt(2, tsKvEntity.getKey());
ps.setLong(3, tsKvEntity.getTs());
if (tsKvEntity.getBooleanValue() != null) {
ps.setBoolean(5, tsKvEntity.getBooleanValue());
ps.setBoolean(10, tsKvEntity.getBooleanValue());
ps.setBoolean(4, tsKvEntity.getBooleanValue());
ps.setBoolean(9, tsKvEntity.getBooleanValue());
} else {
ps.setNull(5, Types.BOOLEAN);
ps.setNull(10, Types.BOOLEAN);
ps.setNull(4, Types.BOOLEAN);
ps.setNull(9, Types.BOOLEAN);
}
ps.setString(6, replaceNullChars(tsKvEntity.getStrValue()));
ps.setString(11, replaceNullChars(tsKvEntity.getStrValue()));
ps.setString(5, replaceNullChars(tsKvEntity.getStrValue()));
ps.setString(10, replaceNullChars(tsKvEntity.getStrValue()));
if (tsKvEntity.getLongValue() != null) {
ps.setLong(7, tsKvEntity.getLongValue());
ps.setLong(12, tsKvEntity.getLongValue());
ps.setLong(6, tsKvEntity.getLongValue());
ps.setLong(11, tsKvEntity.getLongValue());
} else {
ps.setNull(7, Types.BIGINT);
ps.setNull(12, Types.BIGINT);
ps.setNull(6, Types.BIGINT);
ps.setNull(11, Types.BIGINT);
}
if (tsKvEntity.getDoubleValue() != null) {
ps.setDouble(8, tsKvEntity.getDoubleValue());
ps.setDouble(13, tsKvEntity.getDoubleValue());
ps.setDouble(7, tsKvEntity.getDoubleValue());
ps.setDouble(12, tsKvEntity.getDoubleValue());
} else {
ps.setNull(8, Types.DOUBLE);
ps.setNull(13, Types.DOUBLE);
ps.setNull(7, Types.DOUBLE);
ps.setNull(12, Types.DOUBLE);
}
ps.setString(9, replaceNullChars(tsKvEntity.getJsonValue()));
ps.setString(14, replaceNullChars(tsKvEntity.getJsonValue()));
ps.setString(8, replaceNullChars(tsKvEntity.getJsonValue()));
ps.setString(13, replaceNullChars(tsKvEntity.getJsonValue()));
}
@Override

25
dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java

@ -36,7 +36,7 @@ public class AggregationRepository {
public static final String FIND_SUM = "findSum";
public static final String FIND_COUNT = "findCount";
public static final String FROM_WHERE_CLAUSE = "FROM tenant_ts_kv tskv WHERE tskv.tenant_id = cast(:tenantId AS uuid) AND tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.tenant_id, tskv.entity_id, tskv.key, tsBucket";
public static final String FROM_WHERE_CLAUSE = "FROM ts_kv tskv WHERE tskv.entity_id = cast(:entityId AS uuid) AND tskv.key= cast(:entityKey AS int) AND tskv.ts > :startTs AND tskv.ts <= :endTs GROUP BY tskv.entity_id, tskv.key, tsBucket ORDER BY tskv.entity_id, tskv.key, tsBucket";
public static final String FIND_AVG_QUERY = "SELECT time_bucket(:timeBucket, tskv.ts) AS tsBucket, :timeBucket AS interval, SUM(COALESCE(tskv.long_v, 0)) AS longValue, SUM(COALESCE(tskv.dbl_v, 0.0)) AS doubleValue, SUM(CASE WHEN tskv.long_v IS NULL THEN 0 ELSE 1 END) AS longCountValue, SUM(CASE WHEN tskv.dbl_v IS NULL THEN 0 ELSE 1 END) AS doubleCountValue, null AS strValue, 'AVG' AS aggType ";
@ -52,43 +52,42 @@ public class AggregationRepository {
private EntityManager entityManager;
@Async
public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
public CompletableFuture<List<TimescaleTsKvEntity>> findAvg(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
@SuppressWarnings("unchecked")
List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG);
List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_AVG);
return CompletableFuture.supplyAsync(() -> resultList);
}
@Async
public CompletableFuture<List<TimescaleTsKvEntity>> findMax(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
public CompletableFuture<List<TimescaleTsKvEntity>> findMax(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
@SuppressWarnings("unchecked")
List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX);
List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MAX);
return CompletableFuture.supplyAsync(() -> resultList);
}
@Async
public CompletableFuture<List<TimescaleTsKvEntity>> findMin(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
public CompletableFuture<List<TimescaleTsKvEntity>> findMin(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
@SuppressWarnings("unchecked")
List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN);
List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_MIN);
return CompletableFuture.supplyAsync(() -> resultList);
}
@Async
public CompletableFuture<List<TimescaleTsKvEntity>> findSum(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
public CompletableFuture<List<TimescaleTsKvEntity>> findSum(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
@SuppressWarnings("unchecked")
List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM);
List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_SUM);
return CompletableFuture.supplyAsync(() -> resultList);
}
@Async
public CompletableFuture<List<TimescaleTsKvEntity>> findCount(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
public CompletableFuture<List<TimescaleTsKvEntity>> findCount(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs) {
@SuppressWarnings("unchecked")
List<TimescaleTsKvEntity> resultList = getResultList(tenantId, entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT);
List<TimescaleTsKvEntity> resultList = getResultList(entityId, entityKey, timeBucket, startTs, endTs, FIND_COUNT);
return CompletableFuture.supplyAsync(() -> resultList);
}
private List getResultList(UUID tenantId, UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) {
private List getResultList(UUID entityId, int entityKey, long timeBucket, long startTs, long endTs, String query) {
return entityManager.createNamedQuery(query)
.setParameter("tenantId", tenantId)
.setParameter("entityId", entityId)
.setParameter("entityKey", entityKey)
.setParameter("timeBucket", timeBucket)

45
dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java

@ -88,24 +88,23 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
}
@Override
protected ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
protected ListenableFuture<List<TsKvEntry>> findAllAsync(EntityId entityId, ReadTsKvQuery query) {
if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(tenantId, entityId, query);
return findAllAsyncWithLimit(entityId, query);
} else {
long startTs = query.getStartTs();
long endTs = query.getEndTs();
long timeBucket = query.getInterval();
ListenableFuture<List<Optional<TsKvEntry>>> future = findAllAndAggregateAsync(tenantId, entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation());
ListenableFuture<List<Optional<TsKvEntry>>> future = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation());
return getTskvEntriesFuture(future);
}
}
@Override
protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
protected ListenableFuture<List<TsKvEntry>> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) {
String strKey = query.getKey();
Integer keyId = getOrSaveKeyId(strKey);
List<TimescaleTsKvEntity> timescaleTsKvEntities = tsKvRepository.findAllWithLimit(
tenantId.getId(),
entityId.getId(),
keyId,
query.getStartTs(),
@ -117,8 +116,8 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
return Futures.immediateFuture(DaoUtil.convertDataList(timescaleTsKvEntities));
}
private ListenableFuture<List<Optional<TsKvEntry>>> findAllAndAggregateAsync(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) {
CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId(), tenantId.getId());
private ListenableFuture<List<Optional<TsKvEntry>>> findAllAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long timeBucket, Aggregation aggregation) {
CompletableFuture<List<TimescaleTsKvEntity>> listCompletableFuture = switchAggregation(key, startTs, endTs, timeBucket, aggregation, entityId.getId());
SettableFuture<List<TimescaleTsKvEntity>> listenableFuture = SettableFuture.create();
listCompletableFuture.whenComplete((timescaleTsKvEntities, throwable) -> {
if (throwable != null) {
@ -133,7 +132,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
timescaleTsKvEntities.forEach(entity -> {
if (entity != null && entity.isNotEmpty()) {
entity.setEntityId(entityId.getId());
entity.setTenantId(tenantId.getId());
entity.setStrKey(key);
result.add(Optional.of(DaoUtil.getData(entity)));
} else {
@ -167,7 +165,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
String strKey = tsKvEntry.getKey();
Integer keyId = getOrSaveKeyId(strKey);
TimescaleTsKvEntity entity = new TimescaleTsKvEntity();
entity.setTenantId(tenantId.getId());
entity.setEntityId(entityId.getId());
entity.setTs(tsKvEntry.getTs());
entity.setKey(keyId);
@ -197,7 +194,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
Integer keyId = getOrSaveKeyId(strKey);
return service.submit(() -> {
tsKvRepository.delete(
tenantId.getId(),
entityId.getId(),
keyId,
query.getStartTs(),
@ -208,7 +204,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
@Override
public ListenableFuture<Void> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
return getRemoveLatestFuture(tenantId, entityId, query);
return getRemoveLatestFuture(entityId, query);
}
@Override
@ -216,27 +212,26 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
return service.submit(() -> null);
}
private CompletableFuture<List<TimescaleTsKvEntity>> switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId, UUID tenantId) {
private CompletableFuture<List<TimescaleTsKvEntity>> switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId) {
switch (aggregation) {
case AVG:
return findAvg(key, startTs, endTs, timeBucket, entityId, tenantId);
return findAvg(key, startTs, endTs, timeBucket, entityId);
case MAX:
return findMax(key, startTs, endTs, timeBucket, entityId, tenantId);
return findMax(key, startTs, endTs, timeBucket, entityId);
case MIN:
return findMin(key, startTs, endTs, timeBucket, entityId, tenantId);
return findMin(key, startTs, endTs, timeBucket, entityId);
case SUM:
return findSum(key, startTs, endTs, timeBucket, entityId, tenantId);
return findSum(key, startTs, endTs, timeBucket, entityId);
case COUNT:
return findCount(key, startTs, endTs, timeBucket, entityId, tenantId);
return findCount(key, startTs, endTs, timeBucket, entityId);
default:
throw new IllegalArgumentException("Not supported aggregation type: " + aggregation);
}
}
private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
private CompletableFuture<List<TimescaleTsKvEntity>> findCount(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
Integer keyId = getOrSaveKeyId(key);
return aggregationRepository.findCount(
tenantId,
entityId,
keyId,
timeBucket,
@ -244,10 +239,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
endTs);
}
private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
private CompletableFuture<List<TimescaleTsKvEntity>> findSum(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
Integer keyId = getOrSaveKeyId(key);
return aggregationRepository.findSum(
tenantId,
entityId,
keyId,
timeBucket,
@ -255,10 +249,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
endTs);
}
private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
private CompletableFuture<List<TimescaleTsKvEntity>> findMin(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
Integer keyId = getOrSaveKeyId(key);
return aggregationRepository.findMin(
tenantId,
entityId,
keyId,
timeBucket,
@ -266,10 +259,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
endTs);
}
private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
private CompletableFuture<List<TimescaleTsKvEntity>> findMax(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
Integer keyId = getOrSaveKeyId(key);
return aggregationRepository.findMax(
tenantId,
entityId,
keyId,
timeBucket,
@ -277,10 +269,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
endTs);
}
private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId, UUID tenantId) {
private CompletableFuture<List<TimescaleTsKvEntity>> findAvg(String key, long startTs, long endTs, long timeBucket, UUID entityId) {
Integer keyId = getOrSaveKeyId(key);
return aggregationRepository.findAvg(
tenantId,
entityId,
keyId,
timeBucket,

10
dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java

@ -31,12 +31,10 @@ import java.util.UUID;
@TimescaleDBTsDao
public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEntity, TimescaleTsKvCompositeKey> {
@Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.tenantId = :tenantId " +
"AND tskv.entityId = :entityId " +
@Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " +
"AND tskv.key = :entityKey " +
"AND tskv.ts > :startTs AND tskv.ts <= :endTs")
List<TimescaleTsKvEntity> findAllWithLimit(
@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("entityKey") int key,
@Param("startTs") long startTs,
@ -44,12 +42,10 @@ public interface TsKvTimescaleRepository extends CrudRepository<TimescaleTsKvEnt
@Transactional
@Modifying
@Query("DELETE FROM TimescaleTsKvEntity tskv WHERE tskv.tenantId = :tenantId " +
"AND tskv.entityId = :entityId " +
@Query("DELETE FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " +
"AND tskv.key = :entityKey " +
"AND tskv.ts > :startTs AND tskv.ts <= :endTs")
void delete(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
void delete(@Param("entityId") UUID entityId,
@Param("entityKey") int key,
@Param("startTs") long startTs,
@Param("endTs") long endTs);

17
dao/src/main/resources/sql/schema-timescale-idx.sql

@ -1,17 +0,0 @@
--
-- Copyright © 2016-2020 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE INDEX IF NOT EXISTS idx_tenant_ts_kv ON tenant_ts_kv(tenant_id, entity_id, key, ts);

125
dao/src/main/resources/sql/schema-timescale.sql

@ -16,8 +16,7 @@
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
CREATE TABLE IF NOT EXISTS tenant_ts_kv (
tenant_id uuid NOT NULL,
CREATE TABLE IF NOT EXISTS ts_kv (
entity_id uuid NOT NULL,
key int NOT NULL,
ts bigint NOT NULL,
@ -26,7 +25,7 @@ CREATE TABLE IF NOT EXISTS tenant_ts_kv (
long_v bigint,
dbl_v double precision,
json_v json,
CONSTRAINT tenant_ts_kv_pkey PRIMARY KEY (tenant_id, entity_id, key, ts)
CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts)
);
CREATE TABLE IF NOT EXISTS ts_kv_dictionary (
@ -45,4 +44,122 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest (
dbl_v double precision,
json_v json,
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
);
);
CREATE TABLE IF NOT EXISTS tb_schema_settings
(
schema_version bigint NOT NULL,
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000);
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$
BEGIN
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31),
IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
tenant_cursor CURSOR FOR select tenant.id as tenant_id
from tenant;
tenant_id_record varchar;
customer_id_record varchar;
tenant_ttl bigint;
customer_ttl bigint;
deleted_for_entities bigint;
tenant_ttl_ts bigint;
customer_ttl_ts bigint;
BEGIN
OPEN tenant_cursor;
FETCH tenant_cursor INTO tenant_id_record;
WHILE FOUND
LOOP
EXECUTE format(
'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
tenant_id_record, 'TTL') INTO tenant_ttl;
if tenant_ttl IS NULL THEN
tenant_ttl := system_ttl;
END IF;
IF tenant_ttl > 0 THEN
tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint;
deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record;
deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record;
END IF;
FOR customer_id_record IN
SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record
LOOP
EXECUTE format(
'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
customer_id_record, 'TTL') INTO customer_ttl;
IF customer_ttl IS NULL THEN
customer_ttl_ts := tenant_ttl_ts;
ELSE
IF customer_ttl > 0 THEN
customer_ttl_ts :=
(EXTRACT(EPOCH FROM current_timestamp) * 1000 -
customer_ttl::bigint * 1000)::bigint;
END IF;
END IF;
IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN
deleted_for_entities :=
delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record;
deleted_for_entities :=
delete_device_records_from_ts_kv(tenant_id_record, customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record,
customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
END IF;
END LOOP;
FETCH tenant_cursor INTO tenant_id_record;
END LOOP;
END
$$;

271
dao/src/main/resources/sql/schema-ts-psql.sql

@ -14,31 +14,260 @@
-- limitations under the License.
--
CREATE TABLE IF NOT EXISTS ts_kv (
entity_id uuid NOT NULL,
key int NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
json_v json
CREATE TABLE IF NOT EXISTS ts_kv
(
entity_id uuid NOT NULL,
key int NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
json_v json
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS ts_kv_latest (
entity_id uuid NOT NULL,
key int NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
json_v json,
CREATE TABLE IF NOT EXISTS ts_kv_latest
(
entity_id uuid NOT NULL,
key int NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
json_v json,
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
);
CREATE TABLE IF NOT EXISTS ts_kv_dictionary (
key varchar(255) NOT NULL,
CREATE TABLE IF NOT EXISTS ts_kv_dictionary
(
key varchar(255) NOT NULL,
key_id serial UNIQUE,
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
);
);
CREATE TABLE IF NOT EXISTS tb_schema_settings
(
schema_version bigint NOT NULL,
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
);
INSERT INTO tb_schema_settings (schema_version) VALUES (2005000);
CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
max_tenant_ttl bigint;
max_customer_ttl bigint;
max_ttl bigint;
date timestamp;
partition_by_max_ttl_date varchar;
partition_month varchar;
partition_day varchar;
partition_year varchar;
partition varchar;
partition_to_delete varchar;
BEGIN
SELECT max(attribute_kv.long_v)
FROM tenant
INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id
WHERE attribute_kv.attribute_key = 'TTL'
into max_tenant_ttl;
SELECT max(attribute_kv.long_v)
FROM customer
INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id
WHERE attribute_kv.attribute_key = 'TTL'
into max_customer_ttl;
max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
if max_ttl IS NOT NULL AND max_ttl > 0 THEN
date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
IF partition_by_max_ttl_date IS NOT NULL THEN
CASE
WHEN partition_type = 'DAYS' THEN
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
partition_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5);
WHEN partition_type = 'MONTHS' THEN
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
partition_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4);
ELSE
partition_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3);
END CASE;
FOR partition IN SELECT tablename
FROM pg_tables
WHERE schemaname = 'public'
AND tablename like 'ts_kv_' || '%'
AND tablename != 'ts_kv_latest'
AND tablename != 'ts_kv_dictionary'
LOOP
IF partition != partition_by_max_ttl_date THEN
IF partition_year IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 3)::integer < partition_year::integer THEN
partition_to_delete := partition;
ELSE
IF partition_month IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 4)::integer < partition_month::integer THEN
partition_to_delete := partition;
ELSE
IF partition_day IS NOT NULL THEN
IF SPLIT_PART(partition, '_', 5)::integer < partition_day::integer THEN
partition_to_delete := partition;
END IF;
END IF;
END IF;
END IF;
END IF;
END IF;
END IF;
IF partition_to_delete IS NOT NULL THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
EXECUTE format('DROP TABLE %I', partition_to_delete);
deleted := deleted + 1;
END IF;
END LOOP;
END IF;
END IF;
END
$$;
CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS
$$
BEGIN
CASE
WHEN partition_type = 'DAYS' THEN
partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd');
WHEN partition_type = 'MONTHS' THEN
partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM');
WHEN partition_type = 'YEARS' THEN
partition := 'ts_kv_' || to_char(date, 'yyyy');
WHEN partition_type = 'INDEFINITE' THEN
partition := NULL;
ELSE
partition := NULL;
END CASE;
IF partition IS NOT NULL THEN
IF NOT EXISTS(SELECT
FROM pg_tables
WHERE schemaname = 'public'
AND tablename = partition) THEN
partition := NULL;
RAISE NOTICE 'Failed to found partition by ttl';
END IF;
END IF;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$
BEGIN
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(device.id) as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(asset.id) as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id varchar, customer_id varchar, ttl bigint,
OUT deleted bigint) AS
$$
BEGIN
EXECUTE format(
'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT to_uuid(customer.id) as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted',
tenant_id, customer_id, ttl) into deleted;
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid varchar(31),
IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
tenant_cursor CURSOR FOR select tenant.id as tenant_id
from tenant;
tenant_id_record varchar;
customer_id_record varchar;
tenant_ttl bigint;
customer_ttl bigint;
deleted_for_entities bigint;
tenant_ttl_ts bigint;
customer_ttl_ts bigint;
BEGIN
OPEN tenant_cursor;
FETCH tenant_cursor INTO tenant_id_record;
WHILE FOUND
LOOP
EXECUTE format(
'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
tenant_id_record, 'TTL') INTO tenant_ttl;
if tenant_ttl IS NULL THEN
tenant_ttl := system_ttl;
END IF;
IF tenant_ttl > 0 THEN
tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint;
deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record;
deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record;
END IF;
FOR customer_id_record IN
SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record
LOOP
EXECUTE format(
'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L',
customer_id_record, 'TTL') INTO customer_ttl;
IF customer_ttl IS NULL THEN
customer_ttl_ts := tenant_ttl_ts;
ELSE
IF customer_ttl > 0 THEN
customer_ttl_ts :=
(EXTRACT(EPOCH FROM current_timestamp) * 1000 -
customer_ttl::bigint * 1000)::bigint;
END IF;
END IF;
IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN
deleted_for_entities :=
delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record;
deleted_for_entities :=
delete_device_records_from_ts_kv(tenant_id_record, customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record,
customer_id_record,
customer_ttl_ts);
deleted := deleted + deleted_for_entities;
RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record;
END IF;
END LOOP;
FETCH tenant_cursor INTO tenant_id_record;
END LOOP;
END
$$;

2
dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java

@ -44,7 +44,7 @@ public class SqlDaoServiceTestSuite {
// @ClassRule
// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
// Arrays.asList("sql/schema-timescale.sql", "sql/schema-timescale-idx.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
// Arrays.asList("sql/schema-timescale.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
// "sql/timescale/drop-all-tables.sql",
// "sql-test.properties"
// );

8
dao/src/test/java/org/thingsboard/server/dao/nosql/RateLimitedResultSetFutureTest.java

@ -119,7 +119,7 @@ public class RateLimitedResultSetFutureTest {
resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
Row actualRow = transform.get();
assertSame(row, actualRow);
@ -132,7 +132,7 @@ public class RateLimitedResultSetFutureTest {
when(rateLimiter.acquireAsync()).thenReturn(Futures.immediateFuture(null));
when(session.executeAsync(statement)).thenThrow(new UnsupportedFeatureException(ProtocolVersion.V3, "hjg"));
resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
try {
transform.get();
fail();
@ -156,7 +156,7 @@ public class RateLimitedResultSetFutureTest {
when(realFuture.get()).thenThrow(new ExecutionException("Fail", new TimeoutException("timeout")));
resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
try {
transform.get();
fail();
@ -177,7 +177,7 @@ public class RateLimitedResultSetFutureTest {
when(rateLimiter.acquireAsync()).thenReturn(future);
resultSetFuture = new RateLimitedResultSetFuture(session, rateLimiter, statement);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one);
ListenableFuture<Row> transform = Futures.transform(resultSetFuture, ResultSet::one, MoreExecutors.directExecutor());
// TimeUnit.MILLISECONDS.sleep(200);
future.cancel(false);
latch.countDown();

2
dao/src/test/resources/sql/timescale/drop-all-tables.sql

@ -12,7 +12,7 @@ DROP TABLE IF EXISTS event;
DROP TABLE IF EXISTS relation;
DROP TABLE IF EXISTS tb_user;
DROP TABLE IF EXISTS tenant;
DROP TABLE IF EXISTS tenant_ts_kv;
DROP TABLE IF EXISTS ts_kv;
DROP TABLE IF EXISTS ts_kv_latest;
DROP TABLE IF EXISTS user_credentials;
DROP TABLE IF EXISTS widget_type;

7
docker/README.md

@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env`
**NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.cassandra.yml` for details).
Execute the following command to create log folders for the services and chown of these folders to the docker container users.
To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access):
`
$ ./docker-create-log-folders.sh
`
Execute the following command to run installation:
`

24
docker/docker-create-log-folders.sh

@ -0,0 +1,24 @@
#!/bin/bash
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
mkdir -p tb-node/log/ && sudo chown -R 799:799 tb-node/log/
mkdir -p tb-transports/coap/log && sudo chown -R 799:799 tb-transports/coap/log
mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log
mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log

2
k8s/database-setup.yml

@ -39,5 +39,5 @@ spec:
volumeMounts:
- mountPath: /config
name: tb-node-config
command: ['sh', '-c', 'while [ ! -f /install-finished ]; do sleep 2; done;']
command: ['sh', '-c', 'while [ ! -f /tmp/install-finished ]; do sleep 2; done;']
restartPolicy: Never

4
k8s/k8s-delete-all.sh

@ -15,4 +15,6 @@
# limitations under the License.
#
kubectl -n thingsboard delete svc,sts,deploy,pv,pvc,cm,po,ing --all
kubectl -n thingsboard delete svc,sts,deploy,cm,po,ing --all
kubectl -n thingsboard get pvc --no-headers=true | awk '//{print $1}' | xargs kubectl -n thingsboard delete --ignore-not-found=true pvc

2
k8s/k8s-install-tb.sh

@ -22,7 +22,7 @@ function installTb() {
kubectl apply -f tb-node-configmap.yml
kubectl apply -f database-setup.yml &&
kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s &&
kubectl exec tb-db-setup -- sh -c 'export INSTALL_TB=true; export LOAD_DEMO='"$loadDemo"'; start-tb-node.sh; touch /install-finished;'
kubectl exec tb-db-setup -- sh -c 'export INSTALL_TB=true; export LOAD_DEMO='"$loadDemo"'; start-tb-node.sh; touch /tmp/install-finished;'
kubectl delete pod tb-db-setup

2
k8s/k8s-upgrade-tb.sh

@ -38,6 +38,6 @@ fi
kubectl apply -f database-setup.yml &&
kubectl wait --for=condition=Ready pod/tb-db-setup --timeout=120s &&
kubectl exec tb-db-setup -- sh -c 'export UPGRADE_TB=true; export FROM_VERSION='"$fromVersion"'; start-tb-node.sh; touch /install-finished;'
kubectl exec tb-db-setup -- sh -c 'export UPGRADE_TB=true; export FROM_VERSION='"$fromVersion"'; start-tb-node.sh; touch /tmp/install-finished;'
kubectl delete pod tb-db-setup

2
k8s/postgres.yml

@ -58,6 +58,8 @@ spec:
env:
- name: POSTGRES_DB
value: "thingsboard"
- name: POSTGRES_PASSWORD
value: "postgres"
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
volumeMounts:

4
msa/js-executor/docker/Dockerfile

@ -14,7 +14,7 @@
# limitations under the License.
#
FROM debian:stretch
FROM thingsboard/base
COPY start-js-executor.sh ${pkg.name}.deb /tmp/
@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
RUN update-rc.d ${pkg.name} disable
USER ${pkg.user}
CMD ["start-js-executor.sh"]

4
msa/js-executor/docker/start-js-executor.sh

@ -26,4 +26,6 @@ identity=${pkg.name}
source "${CONF_FOLDER}/${configfile}"
su -s /bin/sh -c "$mainfile"
cd ${pkg.installFolder}/bin
exec /bin/sh -c "$mainfile"

1
msa/js-executor/pom.xml

@ -36,7 +36,6 @@
<main.dir>${basedir}/../..</main.dir>
<pkg.name>tb-js-executor</pkg.name>
<docker.name>tb-js-executor</docker.name>
<pkg.user>thingsboard</pkg.user>
<pkg.unixLogFolder>/var/log/${pkg.name}</pkg.unixLogFolder>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
<pkg.linux.dist>${project.build.directory}/package/linux</pkg.linux.dist>

4
msa/tb-node/docker/Dockerfile

@ -25,4 +25,8 @@ RUN dpkg -i /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :
RUN chown -R ${pkg.user}:${pkg.user} /tmp
USER ${pkg.user}
CMD ["start-tb-node.sh"]

4
msa/tb-node/docker/start-tb-node.sh

@ -18,12 +18,14 @@
CONF_FOLDER="/config"
jarfile=${pkg.installFolder}/bin/${pkg.name}.jar
configfile=${pkg.name}.conf
run_user=${pkg.name}
run_user=${pkg.user}
source "${CONF_FOLDER}/${configfile}"
export LOADER_PATH=/config,${LOADER_PATH}
cd ${pkg.installFolder}/bin
if [ "$INSTALL_TB" == "true" ]; then
if [ "$LOAD_DEMO" == "true" ]; then

1
msa/tb-node/pom.xml

@ -36,7 +36,6 @@
<main.dir>${basedir}/../..</main.dir>
<pkg.name>thingsboard</pkg.name>
<docker.name>tb-node</docker.name>
<pkg.user>thingsboard</pkg.user>
<pkg.unixLogFolder>/var/log/${pkg.name}</pkg.unixLogFolder>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
</properties>

1
msa/tb/pom.xml

@ -38,7 +38,6 @@
<tb.docker.name>tb</tb.docker.name>
<tb-postgres.docker.name>tb-postgres</tb-postgres.docker.name>
<tb-cassandra.docker.name>tb-cassandra</tb-cassandra.docker.name>
<pkg.user>thingsboard</pkg.user>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
<pkg.upgradeVersion>2.4.2</pkg.upgradeVersion>
</properties>

2
msa/transport/coap/docker/Dockerfile

@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
RUN update-rc.d ${pkg.name} disable
USER ${pkg.user}
CMD ["start-tb-coap-transport.sh"]

2
msa/transport/coap/docker/start-tb-coap-transport.sh

@ -25,6 +25,8 @@ export LOADER_PATH=/config,${LOADER_PATH}
echo "Starting '${project.name}' ..."
cd ${pkg.installFolder}/bin
exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.coap.ThingsboardCoapTransportApplication \
-Dspring.jpa.hibernate.ddl-auto=none \
-Dlogging.config=/config/logback.xml \

1
msa/transport/coap/pom.xml

@ -36,7 +36,6 @@
<main.dir>${basedir}/../../..</main.dir>
<pkg.name>tb-coap-transport</pkg.name>
<docker.name>tb-coap-transport</docker.name>
<pkg.user>thingsboard</pkg.user>
<pkg.logFolder>/var/log/${pkg.name}</pkg.logFolder>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
</properties>

2
msa/transport/http/docker/Dockerfile

@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
RUN update-rc.d ${pkg.name} disable
USER ${pkg.user}
CMD ["start-tb-http-transport.sh"]

2
msa/transport/http/docker/start-tb-http-transport.sh

@ -25,6 +25,8 @@ export LOADER_PATH=/config,${LOADER_PATH}
echo "Starting '${project.name}' ..."
cd ${pkg.installFolder}/bin
exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.http.ThingsboardHttpTransportApplication \
-Dspring.jpa.hibernate.ddl-auto=none \
-Dlogging.config=/config/logback.xml \

1
msa/transport/http/pom.xml

@ -36,7 +36,6 @@
<main.dir>${basedir}/../../..</main.dir>
<pkg.name>tb-http-transport</pkg.name>
<docker.name>tb-http-transport</docker.name>
<pkg.user>thingsboard</pkg.user>
<pkg.logFolder>/var/log/${pkg.name}</pkg.logFolder>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
</properties>

2
msa/transport/mqtt/docker/Dockerfile

@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
RUN update-rc.d ${pkg.name} disable
USER ${pkg.user}
CMD ["start-tb-mqtt-transport.sh"]

2
msa/transport/mqtt/docker/start-tb-mqtt-transport.sh

@ -25,6 +25,8 @@ export LOADER_PATH=/config,${LOADER_PATH}
echo "Starting '${project.name}' ..."
cd ${pkg.installFolder}/bin
exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.mqtt.ThingsboardMqttTransportApplication \
-Dspring.jpa.hibernate.ddl-auto=none \
-Dlogging.config=/config/logback.xml \

1
msa/transport/mqtt/pom.xml

@ -36,7 +36,6 @@
<main.dir>${basedir}/../../..</main.dir>
<pkg.name>tb-mqtt-transport</pkg.name>
<docker.name>tb-mqtt-transport</docker.name>
<pkg.user>thingsboard</pkg.user>
<pkg.logFolder>/var/log/${pkg.name}</pkg.logFolder>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
</properties>

4
msa/web-ui/docker/Dockerfile

@ -14,7 +14,7 @@
# limitations under the License.
#
FROM debian:stretch
FROM thingsboard/base
COPY start-web-ui.sh ${pkg.name}.deb /tmp/
@ -25,4 +25,6 @@ RUN dpkg -i /tmp/${pkg.name}.deb
RUN update-rc.d ${pkg.name} disable
USER ${pkg.user}
CMD ["start-web-ui.sh"]

4
msa/web-ui/docker/start-web-ui.sh

@ -26,4 +26,6 @@ identity=${pkg.name}
source "${CONF_FOLDER}/${configfile}"
su -s /bin/sh -c "$mainfile"
cd ${pkg.installFolder}/bin
exec /bin/sh -c "$mainfile"

1
msa/web-ui/pom.xml

@ -36,7 +36,6 @@
<main.dir>${basedir}/../..</main.dir>
<pkg.name>tb-web-ui</pkg.name>
<docker.name>tb-web-ui</docker.name>
<pkg.user>thingsboard</pkg.user>
<pkg.unixLogFolder>/var/log/${pkg.name}</pkg.unixLogFolder>
<pkg.installFolder>/usr/share/${pkg.name}</pkg.installFolder>
<pkg.linux.dist>${project.build.directory}/package/linux</pkg.linux.dist>

8
netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java

@ -407,8 +407,12 @@ final class MqttClientImpl implements MqttClient {
}
private MqttMessageIdVariableHeader getNewMessageId() {
this.nextMessageId.compareAndSet(0xffff, 1);
return MqttMessageIdVariableHeader.from(this.nextMessageId.getAndIncrement());
int messageId;
synchronized (this.nextMessageId) {
this.nextMessageId.compareAndSet(0xffff, 1);
messageId = this.nextMessageId.getAndIncrement();
}
return MqttMessageIdVariableHeader.from(messageId);
}
private Future<Void> createSubscription(String topic, MqttHandler handler, boolean once, MqttQoS qos) {

5
pom.xml

@ -29,6 +29,7 @@
<properties>
<main.dir>${basedir}</main.dir>
<pkg.user>thingsboard</pkg.user>
<spring-boot.version>2.1.3.RELEASE</spring-boot.version>
<spring.version>5.1.5.RELEASE</spring.version>
<spring-security.version>5.1.4.RELEASE</spring-security.version>
@ -44,7 +45,7 @@
<cassandra.version>3.6.0</cassandra.version>
<cassandra-unit.version>3.5.0.1</cassandra-unit.version>
<takari-cpsuite.version>1.2.7</takari-cpsuite.version>
<guava.version>21.0</guava.version>
<guava.version>28.2-jre</guava.version>
<caffeine.version>2.6.1</caffeine.version>
<commons-lang3.version>3.4</commons-lang3.version>
<commons-validator.version>1.6</commons-validator.version>
@ -63,7 +64,7 @@
<mail.version>1.4.3</mail.version>
<curator.version>4.2.0</curator.version>
<zookeeper.version>3.5.5</zookeeper.version>
<protobuf.version>3.9.1</protobuf.version>
<protobuf.version>3.11.4</protobuf.version>
<grpc.version>1.22.1</grpc.version>
<lombok.version>1.16.18</lombok.version>
<paho.client.version>1.1.0</paho.client.version>

5
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractRelationActionNode.java

@ -20,6 +20,7 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -54,9 +55,9 @@ import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.thingsboard.common.util.DonAsynchron.withCallback;
import static org.thingsboard.rule.engine.api.TbRelationTypes.FAILURE;
import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS;
import static org.thingsboard.common.util.DonAsynchron.withCallback;
@Slf4j
public abstract class TbAbstractRelationActionNode<C extends TbAbstractRelationActionNodeConfiguration> implements TbNode {
@ -86,7 +87,7 @@ public abstract class TbAbstractRelationActionNode<C extends TbAbstractRelationA
}
protected ListenableFuture<RelationContainer> processEntityRelationAction(TbContext ctx, TbMsg msg) {
return Futures.transformAsync(getEntity(ctx, msg), entityContainer -> doProcessEntityRelationAction(ctx, msg, entityContainer));
return Futures.transformAsync(getEntity(ctx, msg), entityContainer -> doProcessEntityRelationAction(ctx, msg, entityContainer), MoreExecutors.directExecutor());
}
protected abstract boolean createEntityIfNotExists();

7
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java

@ -18,12 +18,13 @@ package org.thingsboard.rule.engine.action;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.rule.engine.api.util.TbNodeUtils;
import org.thingsboard.rule.engine.api.RuleNode;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.rule.engine.api.util.TbNodeUtils;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.alarm.AlarmStatus;
import org.thingsboard.server.common.data.plugin.ComponentType;
@ -80,8 +81,8 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode<TbClearAlarmNodeConfig
}
alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK);
return Futures.immediateFuture(new AlarmResult(false, false, true, alarm));
});
});
}, MoreExecutors.directExecutor());
}, MoreExecutors.directExecutor());
}, ctx.getDbCallbackExecutor());
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save