Igor Kulikov
6 years ago
2303 changed files with 184288 additions and 156146 deletions
@ -0,0 +1,60 @@ |
|||
--- |
|||
name: "\U0001F41E Bug report" |
|||
about: Create a report to help us improve |
|||
title: "[Bug] " |
|||
labels: bug |
|||
assignees: ashvayka, vvlladd28 |
|||
|
|||
--- |
|||
|
|||
**Describe the bug** |
|||
A clear and concise description of what the bug is. |
|||
|
|||
**Your Server Environment** |
|||
<!-- 🔅🔅🔅🔅🔅🔅🔅 Choose one of the following or write your own 🔅🔅🔅🔅🔅🔅🔅--> |
|||
* demo.thingsboard.io |
|||
* cloud.thingsboard.io |
|||
* own setup |
|||
* cloud or local infrastructure or docker deployment |
|||
* ThingsBoard Version |
|||
* OS Name and Version |
|||
|
|||
**Your Client Environment** |
|||
<!-- 🔅🔅🔅🔅🔅🔅🔅 Choose one of the following or write your own 🔅🔅🔅🔅🔅🔅🔅--> |
|||
**Desktop (please complete the following information):** |
|||
|
|||
* OS: [e.g. iOS] |
|||
* Browser [e.g. chrome, safari] |
|||
* Version [e.g. 22] |
|||
|
|||
**Smartphone (please complete the following information):** |
|||
* Device: [e.g. iPhone6] |
|||
* OS: [e.g. iOS8.1] |
|||
* Browser [e.g. stock browser, safari] |
|||
* Version [e.g. 22] |
|||
|
|||
**Your Device** |
|||
|
|||
* Connectivity |
|||
* MQTT |
|||
* HTTP |
|||
* CoAP |
|||
* Gateway |
|||
* Integration: (Specify name) |
|||
* Device vendor and model |
|||
|
|||
**To Reproduce** |
|||
Steps to reproduce the behavior: |
|||
1. Go to '...' |
|||
2. Click on '....' |
|||
3. Scroll down to '....' |
|||
4. See error |
|||
|
|||
**Expected behavior** |
|||
A clear and concise description of what you expected to happen. |
|||
|
|||
**Screenshots** |
|||
If applicable, add screenshots to help explain your problem. |
|||
|
|||
**Additional context** |
|||
Add any other context about the problem here. |
|||
@ -0,0 +1,20 @@ |
|||
--- |
|||
name: Feature request |
|||
about: Suggest an idea for this project |
|||
title: "[Feature Request]" |
|||
labels: feature |
|||
assignees: ikulikov |
|||
|
|||
--- |
|||
|
|||
**Is your feature request related to a problem? Please describe.** |
|||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] |
|||
|
|||
**Describe the solution you'd like** |
|||
A clear and concise description of what you want to happen. |
|||
|
|||
**Describe alternatives you've considered** |
|||
A clear and concise description of any alternative solutions or features you've considered. |
|||
|
|||
**Additional context** |
|||
Add any other context or screenshots about the feature request here. |
|||
@ -0,0 +1,25 @@ |
|||
--- |
|||
name: Question |
|||
about: Describe your questions in details |
|||
title: "[Question] your title here" |
|||
labels: question |
|||
assignees: ashvayka |
|||
|
|||
--- |
|||
|
|||
**Component** |
|||
|
|||
<!-- Choose one of the following and delete all others. --> |
|||
* UI |
|||
* Rule Engine |
|||
* Installation |
|||
* Generic |
|||
|
|||
**Description** |
|||
A clear and concise details. |
|||
|
|||
**Environment** |
|||
<!-- Add information about your environment and ThingsBoard version if applicable --> |
|||
* OS: name and version |
|||
* ThingsBoard: version |
|||
* Browser: name and version |
|||
@ -1,12 +0,0 @@ |
|||
before_install: |
|||
- sudo rm -f /etc/mavenrc |
|||
- export M2_HOME=/usr/local/maven |
|||
- export MAVEN_OPTS="-Dmaven.repo.local=$HOME/.m2/repository -Xms1024m -Xmx3072m" |
|||
- export HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false |
|||
jdk: |
|||
- openjdk8 |
|||
language: java |
|||
sudo: required |
|||
services: |
|||
- docker |
|||
script: mvn clean verify -Ddockerfile.skip=false |
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,878 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS |
|||
$$ |
|||
BEGIN |
|||
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || |
|||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); |
|||
END; |
|||
$$ LANGUAGE plpgsql; |
|||
|
|||
CREATE OR REPLACE FUNCTION extract_ts(uuid UUID) RETURNS BIGINT AS |
|||
$$ |
|||
DECLARE |
|||
bytes bytea; |
|||
BEGIN |
|||
bytes := uuid_send(uuid); |
|||
RETURN |
|||
( |
|||
( |
|||
(get_byte(bytes, 0)::bigint << 24) | |
|||
(get_byte(bytes, 1)::bigint << 16) | |
|||
(get_byte(bytes, 2)::bigint << 8) | |
|||
(get_byte(bytes, 3)::bigint << 0) |
|||
) + ( |
|||
((get_byte(bytes, 4)::bigint << 8 | |
|||
get_byte(bytes, 5)::bigint)) << 32 |
|||
) + ( |
|||
(((get_byte(bytes, 6)::bigint & 15) << 8 | get_byte(bytes, 7)::bigint) & 4095) << 48 |
|||
) - 122192928000000000 |
|||
) / 10000::double precision; |
|||
END |
|||
$$ LANGUAGE plpgsql |
|||
IMMUTABLE |
|||
PARALLEL SAFE |
|||
RETURNS NULL ON NULL INPUT; |
|||
|
|||
|
|||
CREATE OR REPLACE FUNCTION column_type_to_uuid(table_name varchar, column_name varchar) RETURNS VOID |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
execute format('ALTER TABLE %s RENAME COLUMN %s TO old_%s;', table_name, column_name, column_name); |
|||
execute format('ALTER TABLE %s ADD COLUMN %s UUID;', table_name, column_name); |
|||
execute format('UPDATE %s SET %s = to_uuid(old_%s) WHERE old_%s is not null;', table_name, column_name, column_name, column_name); |
|||
execute format('ALTER TABLE %s DROP COLUMN old_%s;', table_name, column_name); |
|||
END; |
|||
$$; |
|||
|
|||
CREATE OR REPLACE FUNCTION get_column_type(table_name varchar, column_name varchar, OUT data_type varchar) RETURNS varchar |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
execute (format('SELECT data_type from information_schema.columns where table_name = %L and column_name = %L', |
|||
table_name, column_name)) INTO data_type; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
CREATE OR REPLACE PROCEDURE drop_all_idx() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
DROP INDEX IF EXISTS idx_alarm_originator_alarm_type; |
|||
DROP INDEX IF EXISTS idx_alarm_originator_created_time; |
|||
DROP INDEX IF EXISTS idx_alarm_tenant_created_time; |
|||
DROP INDEX IF EXISTS idx_event_type_entity_id; |
|||
DROP INDEX IF EXISTS idx_relation_to_id; |
|||
DROP INDEX IF EXISTS idx_relation_from_id; |
|||
DROP INDEX IF EXISTS idx_device_customer_id; |
|||
DROP INDEX IF EXISTS idx_device_customer_id_and_type; |
|||
DROP INDEX IF EXISTS idx_device_type; |
|||
DROP INDEX IF EXISTS idx_asset_customer_id; |
|||
DROP INDEX IF EXISTS idx_asset_customer_id_and_type; |
|||
DROP INDEX IF EXISTS idx_asset_type; |
|||
DROP INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts; |
|||
END; |
|||
$$; |
|||
|
|||
CREATE OR REPLACE PROCEDURE create_all_idx() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
CREATE INDEX IF NOT EXISTS idx_alarm_originator_alarm_type ON alarm(originator_id, type, start_ts DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_alarm_originator_created_time ON alarm(originator_id, created_time DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_alarm_tenant_created_time ON alarm(tenant_id, created_time DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_event_type_entity_id ON event(tenant_id, event_type, entity_type, entity_id); |
|||
CREATE INDEX IF NOT EXISTS idx_relation_to_id ON relation(relation_type_group, to_type, to_id); |
|||
CREATE INDEX IF NOT EXISTS idx_relation_from_id ON relation(relation_type_group, from_type, from_id); |
|||
CREATE INDEX IF NOT EXISTS idx_device_customer_id ON device(tenant_id, customer_id); |
|||
CREATE INDEX IF NOT EXISTS idx_device_customer_id_and_type ON device(tenant_id, customer_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_device_type ON device(tenant_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id); |
|||
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- admin_settings |
|||
CREATE OR REPLACE PROCEDURE update_admin_settings() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'admin_settings'; |
|||
column_id varchar := 'id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE admin_settings DROP CONSTRAINT admin_settings_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE admin_settings ADD CONSTRAINT admin_settings_pkey PRIMARY KEY (id); |
|||
ALTER TABLE admin_settings ADD COLUMN created_time BIGINT; |
|||
UPDATE admin_settings SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- alarm |
|||
CREATE OR REPLACE PROCEDURE update_alarm() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'alarm'; |
|||
column_id varchar := 'id'; |
|||
column_originator_id varchar := 'originator_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE alarm DROP CONSTRAINT alarm_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE alarm ADD COLUMN created_time BIGINT; |
|||
UPDATE alarm SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE alarm ADD CONSTRAINT alarm_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_originator_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_originator_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_originator_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_originator_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- asset |
|||
CREATE OR REPLACE PROCEDURE update_asset() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'asset'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE asset DROP CONSTRAINT asset_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE asset ADD COLUMN created_time BIGINT; |
|||
UPDATE asset SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE asset ADD CONSTRAINT asset_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE asset DROP CONSTRAINT asset_name_unq_key; |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
ALTER TABLE asset ADD CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- attribute_kv |
|||
CREATE OR REPLACE PROCEDURE update_attribute_kv() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'attribute_kv'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE attribute_kv DROP CONSTRAINT attribute_kv_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
ALTER TABLE attribute_kv ADD CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_type, entity_id, attribute_type, attribute_key); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- audit_log |
|||
CREATE OR REPLACE PROCEDURE update_audit_log() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'audit_log'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
column_user_id varchar := 'user_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE audit_log DROP CONSTRAINT audit_log_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE audit_log ADD COLUMN created_time BIGINT; |
|||
UPDATE audit_log SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE audit_log ADD CONSTRAINT audit_log_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_user_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_user_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_user_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_user_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- component_descriptor |
|||
CREATE OR REPLACE PROCEDURE update_component_descriptor() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'component_descriptor'; |
|||
column_id varchar := 'id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE component_descriptor DROP CONSTRAINT component_descriptor_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE component_descriptor ADD CONSTRAINT component_descriptor_pkey PRIMARY KEY (id); |
|||
ALTER TABLE component_descriptor ADD COLUMN created_time BIGINT; |
|||
UPDATE component_descriptor SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- customer |
|||
CREATE OR REPLACE PROCEDURE update_customer() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'customer'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE customer DROP CONSTRAINT customer_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE customer ADD CONSTRAINT customer_pkey PRIMARY KEY (id); |
|||
ALTER TABLE customer ADD COLUMN created_time BIGINT; |
|||
UPDATE customer SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- dashboard |
|||
CREATE OR REPLACE PROCEDURE update_dashboard() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'dashboard'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE dashboard DROP CONSTRAINT dashboard_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE dashboard ADD CONSTRAINT dashboard_pkey PRIMARY KEY (id); |
|||
ALTER TABLE dashboard ADD COLUMN created_time BIGINT; |
|||
UPDATE dashboard SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- device |
|||
CREATE OR REPLACE PROCEDURE update_device() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'device'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device DROP CONSTRAINT device_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE device ADD COLUMN created_time BIGINT; |
|||
UPDATE device SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE device ADD CONSTRAINT device_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device DROP CONSTRAINT device_name_unq_key; |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
ALTER TABLE device ADD CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- device_credentials |
|||
CREATE OR REPLACE PROCEDURE update_device_credentials() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'device_credentials'; |
|||
column_id varchar := 'id'; |
|||
column_device_id varchar := 'device_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device_credentials DROP CONSTRAINT device_credentials_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE device_credentials ADD COLUMN created_time BIGINT; |
|||
UPDATE device_credentials SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE device_credentials ADD CONSTRAINT device_credentials_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_device_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device_credentials DROP CONSTRAINT IF EXISTS device_credentials_device_id_unq_key; |
|||
PERFORM column_type_to_uuid(table_name, column_device_id); |
|||
-- remove duplicate credentials with same device_id |
|||
DELETE from device_credentials where id in ( |
|||
select dc.id |
|||
from ( |
|||
SELECT id, device_id, |
|||
ROW_NUMBER() OVER ( |
|||
PARTITION BY |
|||
device_id |
|||
ORDER BY |
|||
created_time DESC |
|||
) row_num |
|||
FROM |
|||
device_credentials |
|||
) as dc |
|||
WHERE dc.row_num > 1 |
|||
); |
|||
ALTER TABLE device_credentials ADD CONSTRAINT device_credentials_device_id_unq_key UNIQUE (device_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_device_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_device_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- event |
|||
CREATE OR REPLACE PROCEDURE update_event() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'event'; |
|||
column_id varchar := 'id'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE event DROP CONSTRAINT event_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE event ADD COLUMN created_time BIGINT; |
|||
UPDATE event SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE event ADD CONSTRAINT event_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
ALTER TABLE event DROP CONSTRAINT event_unq_key; |
|||
|
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
|
|||
ALTER TABLE event ADD CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid); |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- relation |
|||
CREATE OR REPLACE PROCEDURE update_relation() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'relation'; |
|||
column_from_id varchar := 'from_id'; |
|||
column_to_id varchar := 'to_id'; |
|||
BEGIN |
|||
ALTER TABLE relation DROP CONSTRAINT relation_pkey; |
|||
|
|||
data_type := get_column_type(table_name, column_from_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_from_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_from_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_from_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_to_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_to_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_to_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_to_id; |
|||
END IF; |
|||
|
|||
ALTER TABLE relation ADD CONSTRAINT relation_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type); |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- tb_user |
|||
CREATE OR REPLACE PROCEDURE update_tb_user() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'tb_user'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE tb_user DROP CONSTRAINT tb_user_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE tb_user ADD COLUMN created_time BIGINT; |
|||
UPDATE tb_user SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE tb_user ADD CONSTRAINT tb_user_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- tenant |
|||
CREATE OR REPLACE PROCEDURE update_tenant() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'tenant'; |
|||
column_id varchar := 'id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE tenant DROP CONSTRAINT tenant_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE tenant ADD COLUMN created_time BIGINT; |
|||
UPDATE tenant SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE tenant ADD CONSTRAINT tenant_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- user_credentials |
|||
CREATE OR REPLACE PROCEDURE update_user_credentials() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'user_credentials'; |
|||
column_id varchar := 'id'; |
|||
column_user_id varchar := 'user_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE user_credentials DROP CONSTRAINT user_credentials_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE user_credentials ADD COLUMN created_time BIGINT; |
|||
UPDATE user_credentials SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE user_credentials ADD CONSTRAINT user_credentials_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_user_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE user_credentials DROP CONSTRAINT user_credentials_user_id_key; |
|||
ALTER TABLE user_credentials RENAME COLUMN user_id TO old_user_id; |
|||
ALTER TABLE user_credentials ADD COLUMN user_id UUID UNIQUE; |
|||
UPDATE user_credentials SET user_id = to_uuid(old_user_id) WHERE old_user_id is not null; |
|||
ALTER TABLE user_credentials DROP COLUMN old_user_id; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_user_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_user_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- widget_type |
|||
CREATE OR REPLACE PROCEDURE update_widget_type() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'widget_type'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE widget_type DROP CONSTRAINT widget_type_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE widget_type ADD COLUMN created_time BIGINT; |
|||
UPDATE widget_type SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE widget_type ADD CONSTRAINT widget_type_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- widgets_bundle |
|||
CREATE OR REPLACE PROCEDURE update_widgets_bundle() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'widgets_bundle'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE widgets_bundle DROP CONSTRAINT widgets_bundle_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE widgets_bundle ADD COLUMN created_time BIGINT; |
|||
UPDATE widgets_bundle SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE widgets_bundle ADD CONSTRAINT widgets_bundle_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- rule_chain |
|||
CREATE OR REPLACE PROCEDURE update_rule_chain() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'rule_chain'; |
|||
column_id varchar := 'id'; |
|||
column_first_rule_node_id varchar := 'first_rule_node_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE rule_chain DROP CONSTRAINT rule_chain_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE rule_chain ADD COLUMN created_time BIGINT; |
|||
UPDATE rule_chain SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE rule_chain ADD CONSTRAINT rule_chain_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_first_rule_node_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_first_rule_node_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_first_rule_node_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_first_rule_node_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- rule_node |
|||
CREATE OR REPLACE PROCEDURE update_rule_node() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'rule_node'; |
|||
column_id varchar := 'id'; |
|||
column_rule_chain_id varchar := 'rule_chain_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE rule_node DROP CONSTRAINT rule_node_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE rule_node ADD COLUMN created_time BIGINT; |
|||
UPDATE rule_node SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE rule_node ADD CONSTRAINT rule_node_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_rule_chain_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_rule_chain_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_rule_chain_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_rule_chain_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- entity_view |
|||
CREATE OR REPLACE PROCEDURE update_entity_view() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'entity_view'; |
|||
column_id varchar := 'id'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE entity_view DROP CONSTRAINT entity_view_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE entity_view ADD COLUMN created_time BIGINT; |
|||
UPDATE entity_view SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE entity_view ADD CONSTRAINT entity_view_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
CREATE TABLE IF NOT EXISTS ts_kv_latest |
|||
( |
|||
entity_id uuid NOT NULL, |
|||
key int NOT NULL, |
|||
ts bigint NOT NULL, |
|||
bool_v boolean, |
|||
str_v varchar(10000000), |
|||
long_v bigint, |
|||
dbl_v double precision, |
|||
json_v json, |
|||
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS ts_kv_dictionary |
|||
( |
|||
key varchar(255) NOT NULL, |
|||
key_id serial UNIQUE, |
|||
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) |
|||
); |
|||
@ -0,0 +1,79 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RequestMethod; |
|||
import org.springframework.web.bind.annotation.ResponseBody; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AlarmData; |
|||
import org.thingsboard.server.common.data.query.AlarmDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityCountQuery; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.query.EntityQueryService; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api") |
|||
public class EntityQueryController extends BaseController { |
|||
|
|||
@Autowired |
|||
private EntityQueryService entityQueryService; |
|||
|
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/entitiesQuery/count", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public long countEntitiesByQuery(@RequestBody EntityCountQuery query) throws ThingsboardException { |
|||
checkNotNull(query); |
|||
try { |
|||
return this.entityQueryService.countEntitiesByQuery(getCurrentUser(), query); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/entitiesQuery/find", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public PageData<EntityData> findEntityDataByQuery(@RequestBody EntityDataQuery query) throws ThingsboardException { |
|||
checkNotNull(query); |
|||
try { |
|||
return this.entityQueryService.findEntityDataByQuery(getCurrentUser(), query); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/alarmsQuery/find", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public PageData<AlarmData> findAlarmDataByQuery(@RequestBody AlarmDataQuery query) throws ThingsboardException { |
|||
checkNotNull(query); |
|||
try { |
|||
return this.entityQueryService.findAlarmDataByQuery(getCurrentUser(), query); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
} |
|||
@ -1,314 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import com.datastax.driver.core.KeyspaceMetadata; |
|||
import com.datastax.driver.core.exceptions.InvalidQueryException; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.dashboard.DashboardService; |
|||
import org.thingsboard.server.dao.util.NoSqlDao; |
|||
import org.thingsboard.server.service.install.cql.CassandraDbHelper; |
|||
|
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
|
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ASSET; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ASSIGNED_CUSTOMERS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.CONFIGURATION; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.CUSTOMER_ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.DASHBOARD; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.DEVICE; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.END_TS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_TYPE; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_VIEW; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_VIEWS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.KEYS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.NAME; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.SEARCH_TEXT; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.START_TS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.TENANT_ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.TITLE; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; |
|||
|
|||
@Service |
|||
@NoSqlDao |
|||
@Profile("install") |
|||
@Slf4j |
|||
public class CassandraDatabaseUpgradeService extends AbstractCassandraDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { |
|||
|
|||
private static final String SCHEMA_UPDATE_CQL = "schema_update.cql"; |
|||
|
|||
@Autowired |
|||
private DashboardService dashboardService; |
|||
|
|||
@Autowired |
|||
private InstallScripts installScripts; |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
|
|||
switch (fromVersion) { |
|||
case "1.2.3": |
|||
|
|||
log.info("Upgrading Cassandara DataBase from version {} to 1.3.0 ...", fromVersion); |
|||
|
|||
//Dump devices, assets and relations
|
|||
|
|||
cluster.getSession(); |
|||
|
|||
KeyspaceMetadata ks = cluster.getCluster().getMetadata().getKeyspace(cluster.getKeyspaceName()); |
|||
|
|||
log.info("Dumping devices ..."); |
|||
Path devicesDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), DEVICE, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, |
|||
new String[]{"", "", "", "", "", "", "default"}, |
|||
"tb-devices"); |
|||
log.info("Devices dumped."); |
|||
|
|||
log.info("Dumping assets ..."); |
|||
Path assetsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), ASSET, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, |
|||
new String[]{"", "", "", "", "", "", "default"}, |
|||
"tb-assets"); |
|||
log.info("Assets dumped."); |
|||
|
|||
log.info("Dumping relations ..."); |
|||
Path relationsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), "relation", |
|||
new String[]{"from_id", "from_type", "to_id", "to_type", "relation_type", ADDITIONAL_INFO, "relation_type_group"}, |
|||
new String[]{"", "", "", "", "", "", "COMMON"}, |
|||
"tb-relations"); |
|||
log.info("Relations dumped."); |
|||
|
|||
log.info("Updating schema ..."); |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "1.3.0", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
//Restore devices, assets and relations
|
|||
|
|||
log.info("Restoring devices ..."); |
|||
if (devicesDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), DEVICE, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, devicesDump); |
|||
Files.deleteIfExists(devicesDump); |
|||
} |
|||
log.info("Devices restored."); |
|||
|
|||
log.info("Dumping device types ..."); |
|||
Path deviceTypesDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), DEVICE, |
|||
new String[]{TENANT_ID, "type"}, |
|||
new String[]{"", ""}, |
|||
"tb-device-types"); |
|||
if (deviceTypesDump != null) { |
|||
CassandraDbHelper.appendToEndOfLine(deviceTypesDump, "DEVICE"); |
|||
} |
|||
log.info("Device types dumped."); |
|||
log.info("Loading device types ..."); |
|||
if (deviceTypesDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), "entity_subtype", |
|||
new String[]{TENANT_ID, "type", "entity_type"}, deviceTypesDump); |
|||
Files.deleteIfExists(deviceTypesDump); |
|||
} |
|||
log.info("Device types loaded."); |
|||
|
|||
log.info("Restoring assets ..."); |
|||
if (assetsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), ASSET, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, assetsDump); |
|||
Files.deleteIfExists(assetsDump); |
|||
} |
|||
log.info("Assets restored."); |
|||
|
|||
log.info("Dumping asset types ..."); |
|||
Path assetTypesDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), ASSET, |
|||
new String[]{TENANT_ID, "type"}, |
|||
new String[]{"", ""}, |
|||
"tb-asset-types"); |
|||
if (assetTypesDump != null) { |
|||
CassandraDbHelper.appendToEndOfLine(assetTypesDump, "ASSET"); |
|||
} |
|||
log.info("Asset types dumped."); |
|||
log.info("Loading asset types ..."); |
|||
if (assetTypesDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), "entity_subtype", |
|||
new String[]{TENANT_ID, "type", "entity_type"}, assetTypesDump); |
|||
Files.deleteIfExists(assetTypesDump); |
|||
} |
|||
log.info("Asset types loaded."); |
|||
|
|||
log.info("Restoring relations ..."); |
|||
if (relationsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), "relation", |
|||
new String[]{"from_id", "from_type", "to_id", "to_type", "relation_type", ADDITIONAL_INFO, "relation_type_group"}, relationsDump); |
|||
Files.deleteIfExists(relationsDump); |
|||
} |
|||
log.info("Relations restored."); |
|||
|
|||
break; |
|||
case "1.3.0": |
|||
break; |
|||
case "1.3.1": |
|||
|
|||
cluster.getSession(); |
|||
|
|||
ks = cluster.getCluster().getMetadata().getKeyspace(cluster.getKeyspaceName()); |
|||
|
|||
log.info("Dumping dashboards ..."); |
|||
Path dashboardsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), DASHBOARD, |
|||
new String[]{ID, TENANT_ID, CUSTOMER_ID, TITLE, SEARCH_TEXT, ASSIGNED_CUSTOMERS, CONFIGURATION}, |
|||
new String[]{"", "", "", "", "", "", ""}, |
|||
"tb-dashboards", true); |
|||
log.info("Dashboards dumped."); |
|||
|
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "1.4.0", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
log.info("Restoring dashboards ..."); |
|||
if (dashboardsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), DASHBOARD, |
|||
new String[]{ID, TENANT_ID, TITLE, SEARCH_TEXT, CONFIGURATION}, dashboardsDump, true); |
|||
DatabaseHelper.upgradeTo40_assignDashboards(dashboardsDump, dashboardService, false); |
|||
Files.deleteIfExists(dashboardsDump); |
|||
} |
|||
log.info("Dashboards restored."); |
|||
break; |
|||
case "1.4.0": |
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.0.0", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
break; |
|||
|
|||
case "2.0.0": |
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.1.1", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
break; |
|||
|
|||
case "2.1.1": |
|||
|
|||
log.info("Upgrading Cassandra DataBase from version {} to 2.1.2 ...", fromVersion); |
|||
|
|||
cluster.getSession(); |
|||
|
|||
ks = cluster.getCluster().getMetadata().getKeyspace(cluster.getKeyspaceName()); |
|||
|
|||
log.info("Dumping entity views ..."); |
|||
Path entityViewsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), ENTITY_VIEWS, |
|||
new String[]{ID, ENTITY_ID, ENTITY_TYPE, TENANT_ID, CUSTOMER_ID, NAME, TYPE, KEYS, START_TS, END_TS, SEARCH_TEXT, ADDITIONAL_INFO}, |
|||
new String[]{"", "", "", "", "", "", "default", "", "0", "0", "", ""}, |
|||
"tb-entity-views"); |
|||
log.info("Entity views dumped."); |
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.1.2", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
log.info("Restoring entity views ..."); |
|||
if (entityViewsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), ENTITY_VIEW, |
|||
new String[]{ID, ENTITY_ID, ENTITY_TYPE, TENANT_ID, CUSTOMER_ID, NAME, TYPE, KEYS, START_TS, END_TS, SEARCH_TEXT, ADDITIONAL_INFO}, entityViewsDump); |
|||
Files.deleteIfExists(entityViewsDump); |
|||
} |
|||
log.info("Entity views restored."); |
|||
|
|||
break; |
|||
case "2.1.3": |
|||
break; |
|||
case "2.3.0": |
|||
break; |
|||
case "2.3.1": |
|||
log.info("Updating schema ..."); |
|||
String updateDeviceTableStmt = "alter table device add label text"; |
|||
try { |
|||
cluster.getSession().execute(updateDeviceTableStmt); |
|||
Thread.sleep(2500); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
case "2.4.1": |
|||
log.info("Updating schema ..."); |
|||
String updateAssetTableStmt = "alter table asset add label text"; |
|||
try { |
|||
log.info("Updating assets ..."); |
|||
cluster.getSession().execute(updateAssetTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Assets updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
case "2.4.2": |
|||
log.info("Updating schema ..."); |
|||
String updateAlarmTableStmt = "alter table alarm add propagate_relation_types text"; |
|||
try { |
|||
log.info("Updating alarms ..."); |
|||
cluster.getSession().execute(updateAlarmTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Alarms updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
case "2.4.3": |
|||
log.info("Updating schema ..."); |
|||
String updateAttributeKvTableStmt = "alter table attributes_kv_cf add json_v text"; |
|||
try { |
|||
log.info("Updating attributes ..."); |
|||
cluster.getSession().execute(updateAttributeKvTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Attributes updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
|
|||
String updateTenantCoreTableStmt = "alter table tenant add isolated_tb_core boolean"; |
|||
String updateTenantRuleEngineTableStmt = "alter table tenant add isolated_tb_rule_engine boolean"; |
|||
|
|||
try { |
|||
log.info("Updating tenant..."); |
|||
cluster.getSession().execute(updateTenantCoreTableStmt); |
|||
Thread.sleep(2500); |
|||
|
|||
cluster.getSession().execute(updateTenantRuleEngineTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Tenant updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -1,30 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.NoSqlDao; |
|||
|
|||
@Service |
|||
@NoSqlDao |
|||
@Profile("install") |
|||
public class CassandraEntityDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService |
|||
implements EntityDatabaseSchemaService { |
|||
public CassandraEntityDatabaseSchemaService() { |
|||
super("schema-entities.cql"); |
|||
} |
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.NoSqlTsLatestDao; |
|||
|
|||
@Service |
|||
@NoSqlTsLatestDao |
|||
@Profile("install") |
|||
public class CassandraTsLatestDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService |
|||
implements TsLatestDatabaseSchemaService { |
|||
public CassandraTsLatestDatabaseSchemaService() { |
|||
super("schema-ts-latest.cql"); |
|||
} |
|||
} |
|||
@ -0,0 +1,19 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
public interface TsLatestDatabaseSchemaService extends DatabaseSchemaService { |
|||
} |
|||
@ -0,0 +1,327 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
import org.thingsboard.server.dao.cassandra.CassandraCluster; |
|||
import org.thingsboard.server.dao.util.NoSqlAnyDao; |
|||
import org.thingsboard.server.service.install.EntityDatabaseSchemaService; |
|||
|
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
import java.util.Arrays; |
|||
import java.util.List; |
|||
|
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.bigintColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.booleanColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.doubleColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.enumToIntColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.idColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.jsonColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.stringColumn; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@NoSqlAnyDao |
|||
@Slf4j |
|||
public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateService { |
|||
|
|||
@Autowired |
|||
private EntityDatabaseSchemaService entityDatabaseSchemaService; |
|||
|
|||
@Autowired |
|||
protected CassandraCluster cluster; |
|||
|
|||
@Value("${spring.datasource.url}") |
|||
protected String dbUrl; |
|||
|
|||
@Value("${spring.datasource.username}") |
|||
protected String dbUserName; |
|||
|
|||
@Value("${spring.datasource.password}") |
|||
protected String dbPassword; |
|||
|
|||
@Override |
|||
public void migrate() throws Exception { |
|||
log.info("Performing migration of entities data from cassandra to SQL database ..."); |
|||
entityDatabaseSchemaService.createDatabaseSchema(false); |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
conn.setAutoCommit(false); |
|||
for (CassandraToSqlTable table: tables) { |
|||
table.migrateToSql(cluster.getSession(), conn); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Unexpected error during ThingsBoard entities data migration!", e); |
|||
throw e; |
|||
} |
|||
entityDatabaseSchemaService.createDatabaseIndexes(); |
|||
} |
|||
|
|||
private static List<CassandraToSqlTable> tables = Arrays.asList( |
|||
new CassandraToSqlTable("admin_settings", |
|||
idColumn("id"), |
|||
stringColumn("key"), |
|||
stringColumn("json_value")), |
|||
new CassandraToSqlTable("alarm", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("type"), |
|||
idColumn("originator_id"), |
|||
enumToIntColumn("originator_type", EntityType.class), |
|||
stringColumn("severity"), |
|||
stringColumn("status"), |
|||
bigintColumn("start_ts"), |
|||
bigintColumn("end_ts"), |
|||
bigintColumn("ack_ts"), |
|||
bigintColumn("clear_ts"), |
|||
stringColumn("details", "additional_info"), |
|||
booleanColumn("propagate"), |
|||
stringColumn("propagate_relation_types")), |
|||
new CassandraToSqlTable("asset", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
stringColumn("name"), |
|||
stringColumn("type"), |
|||
stringColumn("label"), |
|||
stringColumn("search_text"), |
|||
stringColumn("additional_info")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("asset_name_unq_key")) { |
|||
this.handleUniqueNameViolation(data, "asset"); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("audit_log_by_tenant_id", "audit_log", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("entity_name"), |
|||
idColumn("user_id"), |
|||
stringColumn("user_name"), |
|||
stringColumn("action_type"), |
|||
stringColumn("action_data"), |
|||
stringColumn("action_status"), |
|||
stringColumn("action_failure_details")), |
|||
new CassandraToSqlTable("attributes_kv_cf", "attribute_kv", |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("attribute_type"), |
|||
stringColumn("attribute_key"), |
|||
booleanColumn("bool_v", true), |
|||
stringColumn("str_v"), |
|||
bigintColumn("long_v"), |
|||
doubleColumn("dbl_v"), |
|||
jsonColumn("json_v"), |
|||
bigintColumn("last_update_ts")), |
|||
new CassandraToSqlTable("component_descriptor", |
|||
idColumn("id"), |
|||
stringColumn("type"), |
|||
stringColumn("scope"), |
|||
stringColumn("name"), |
|||
stringColumn("search_text"), |
|||
stringColumn("clazz"), |
|||
stringColumn("configuration_descriptor"), |
|||
stringColumn("actions")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("component_descriptor_clazz_key")) { |
|||
String clazz = this.getColumnData(data, "clazz").getValue(); |
|||
log.warn("Found component_descriptor record with duplicate clazz [{}]. Record will be ignored!", clazz); |
|||
this.ignoreRecord(batchData, data); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("customer", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text"), |
|||
stringColumn("country"), |
|||
stringColumn("state"), |
|||
stringColumn("city"), |
|||
stringColumn("address"), |
|||
stringColumn("address2"), |
|||
stringColumn("zip"), |
|||
stringColumn("phone"), |
|||
stringColumn("email"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("dashboard", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text"), |
|||
stringColumn("assigned_customers"), |
|||
stringColumn("configuration")), |
|||
new CassandraToSqlTable("device", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
stringColumn("name"), |
|||
stringColumn("type"), |
|||
stringColumn("label"), |
|||
stringColumn("search_text"), |
|||
stringColumn("additional_info")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("device_name_unq_key")) { |
|||
this.handleUniqueNameViolation(data, "device"); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("device_credentials", |
|||
idColumn("id"), |
|||
idColumn("device_id"), |
|||
stringColumn("credentials_type"), |
|||
stringColumn("credentials_id"), |
|||
stringColumn("credentials_value")), |
|||
new CassandraToSqlTable("event", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("event_type"), |
|||
stringColumn("event_uid"), |
|||
stringColumn("body"), |
|||
new CassandraToSqlEventTsColumn()), |
|||
new CassandraToSqlTable("relation", |
|||
idColumn("from_id"), |
|||
stringColumn("from_type"), |
|||
idColumn("to_id"), |
|||
stringColumn("to_type"), |
|||
stringColumn("relation_type_group"), |
|||
stringColumn("relation_type"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("user", "tb_user", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
stringColumn("email"), |
|||
stringColumn("search_text"), |
|||
stringColumn("authority"), |
|||
stringColumn("first_name"), |
|||
stringColumn("last_name"), |
|||
stringColumn("additional_info")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("tb_user_email_key")) { |
|||
this.handleUniqueEmailViolation(data); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("tenant", |
|||
idColumn("id"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text"), |
|||
stringColumn("region"), |
|||
stringColumn("country"), |
|||
stringColumn("state"), |
|||
stringColumn("city"), |
|||
stringColumn("address"), |
|||
stringColumn("address2"), |
|||
stringColumn("zip"), |
|||
stringColumn("phone"), |
|||
stringColumn("email"), |
|||
stringColumn("additional_info"), |
|||
booleanColumn("isolated_tb_core"), |
|||
booleanColumn("isolated_tb_rule_engine")), |
|||
new CassandraToSqlTable("user_credentials", |
|||
idColumn("id"), |
|||
idColumn("user_id"), |
|||
booleanColumn("enabled"), |
|||
stringColumn("password"), |
|||
stringColumn("activate_token"), |
|||
stringColumn("reset_token")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("user_credentials_user_id_key")) { |
|||
String id = UUIDConverter.fromString(this.getColumnData(data, "id").getValue()).toString(); |
|||
log.warn("Found user credentials record with duplicate user_id [id:[{}]]. Record will be ignored!", id); |
|||
this.ignoreRecord(batchData, data); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("widget_type", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("bundle_alias"), |
|||
stringColumn("alias"), |
|||
stringColumn("name"), |
|||
stringColumn("descriptor")), |
|||
new CassandraToSqlTable("widgets_bundle", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("alias"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text")), |
|||
new CassandraToSqlTable("rule_chain", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("name"), |
|||
stringColumn("search_text"), |
|||
idColumn("first_rule_node_id"), |
|||
booleanColumn("root"), |
|||
booleanColumn("debug_mode"), |
|||
stringColumn("configuration"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("rule_node", |
|||
idColumn("id"), |
|||
idColumn("rule_chain_id"), |
|||
stringColumn("type"), |
|||
stringColumn("name"), |
|||
booleanColumn("debug_mode"), |
|||
stringColumn("search_text"), |
|||
stringColumn("configuration"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("entity_view", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("name"), |
|||
stringColumn("type"), |
|||
stringColumn("keys"), |
|||
bigintColumn("start_ts"), |
|||
bigintColumn("end_ts"), |
|||
stringColumn("search_text"), |
|||
stringColumn("additional_info")) |
|||
); |
|||
} |
|||
@ -0,0 +1,178 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import com.datastax.oss.driver.api.core.cql.Row; |
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
|
|||
import java.sql.PreparedStatement; |
|||
import java.sql.SQLException; |
|||
import java.sql.Types; |
|||
import java.util.regex.Pattern; |
|||
|
|||
@Data |
|||
public class CassandraToSqlColumn { |
|||
|
|||
private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); |
|||
private static final String EMPTY_STR = ""; |
|||
|
|||
private int index; |
|||
private int sqlIndex; |
|||
private String cassandraColumnName; |
|||
private String sqlColumnName; |
|||
private CassandraToSqlColumnType type; |
|||
private int sqlType; |
|||
private int size; |
|||
private Class<? extends Enum> enumClass; |
|||
private boolean allowNullBoolean = false; |
|||
|
|||
public static CassandraToSqlColumn idColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.ID); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn stringColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.STRING); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn stringColumn(String cassandraColumnName, String sqlColumnName) { |
|||
return new CassandraToSqlColumn(cassandraColumnName, sqlColumnName); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn bigintColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.BIGINT); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn doubleColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.DOUBLE); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn booleanColumn(String name) { |
|||
return booleanColumn(name, false); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn booleanColumn(String name, boolean allowNullBoolean) { |
|||
return new CassandraToSqlColumn(name, name, CassandraToSqlColumnType.BOOLEAN, null, allowNullBoolean); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn jsonColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.JSON); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn enumToIntColumn(String name, Class<? extends Enum> enumClass) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.ENUM_TO_INT, enumClass); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String columnName) { |
|||
this(columnName, columnName, CassandraToSqlColumnType.STRING, null, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String columnName, CassandraToSqlColumnType type) { |
|||
this(columnName, columnName, type, null, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String columnName, CassandraToSqlColumnType type, Class<? extends Enum> enumClass) { |
|||
this(columnName, columnName, type, enumClass, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String cassandraColumnName, String sqlColumnName) { |
|||
this(cassandraColumnName, sqlColumnName, CassandraToSqlColumnType.STRING, null, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String cassandraColumnName, String sqlColumnName, CassandraToSqlColumnType type, |
|||
Class<? extends Enum> enumClass, boolean allowNullBoolean) { |
|||
this.cassandraColumnName = cassandraColumnName; |
|||
this.sqlColumnName = sqlColumnName; |
|||
this.type = type; |
|||
this.enumClass = enumClass; |
|||
this.allowNullBoolean = allowNullBoolean; |
|||
} |
|||
|
|||
public String getColumnValue(Row row) { |
|||
if (row.isNull(index)) { |
|||
if (this.type == CassandraToSqlColumnType.BOOLEAN && !this.allowNullBoolean) { |
|||
return Boolean.toString(false); |
|||
} else { |
|||
return null; |
|||
} |
|||
} else { |
|||
switch (this.type) { |
|||
case ID: |
|||
return UUIDConverter.fromTimeUUID(row.getUuid(index)); |
|||
case DOUBLE: |
|||
return Double.toString(row.getDouble(index)); |
|||
case INTEGER: |
|||
return Integer.toString(row.getInt(index)); |
|||
case FLOAT: |
|||
return Float.toString(row.getFloat(index)); |
|||
case BIGINT: |
|||
return Long.toString(row.getLong(index)); |
|||
case BOOLEAN: |
|||
return Boolean.toString(row.getBoolean(index)); |
|||
case STRING: |
|||
case JSON: |
|||
case ENUM_TO_INT: |
|||
default: |
|||
String value = row.getString(index); |
|||
return this.replaceNullChars(value); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void setColumnValue(PreparedStatement sqlInsertStatement, String value) throws SQLException { |
|||
if (value == null) { |
|||
sqlInsertStatement.setNull(this.sqlIndex, this.sqlType); |
|||
} else { |
|||
switch (this.type) { |
|||
case DOUBLE: |
|||
sqlInsertStatement.setDouble(this.sqlIndex, Double.parseDouble(value)); |
|||
break; |
|||
case INTEGER: |
|||
sqlInsertStatement.setInt(this.sqlIndex, Integer.parseInt(value)); |
|||
break; |
|||
case FLOAT: |
|||
sqlInsertStatement.setFloat(this.sqlIndex, Float.parseFloat(value)); |
|||
break; |
|||
case BIGINT: |
|||
sqlInsertStatement.setLong(this.sqlIndex, Long.parseLong(value)); |
|||
break; |
|||
case BOOLEAN: |
|||
sqlInsertStatement.setBoolean(this.sqlIndex, Boolean.parseBoolean(value)); |
|||
break; |
|||
case ENUM_TO_INT: |
|||
Enum enumVal = Enum.valueOf(this.enumClass, value); |
|||
int intValue = enumVal.ordinal(); |
|||
sqlInsertStatement.setInt(this.sqlIndex, intValue); |
|||
break; |
|||
case JSON: |
|||
case STRING: |
|||
case ID: |
|||
default: |
|||
sqlInsertStatement.setString(this.sqlIndex, value); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
private String replaceNullChars(String strValue) { |
|||
if (strValue != null) { |
|||
return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); |
|||
} |
|||
return strValue; |
|||
} |
|||
|
|||
} |
|||
|
|||
@ -0,0 +1,64 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import lombok.Data; |
|||
|
|||
@Data |
|||
public class CassandraToSqlColumnData { |
|||
|
|||
private String value; |
|||
private String originalValue; |
|||
private int constraintCounter = 0; |
|||
|
|||
public CassandraToSqlColumnData(String value) { |
|||
this.value = value; |
|||
this.originalValue = value; |
|||
} |
|||
|
|||
public int nextContraintCounter() { |
|||
return ++constraintCounter; |
|||
} |
|||
|
|||
public String getNextConstraintStringValue(CassandraToSqlColumn column) { |
|||
int counter = this.nextContraintCounter(); |
|||
String newValue = this.originalValue + counter; |
|||
int overflow = newValue.length() - column.getSize(); |
|||
if (overflow > 0) { |
|||
newValue = this.originalValue.substring(0, this.originalValue.length()-overflow) + counter; |
|||
} |
|||
return newValue; |
|||
} |
|||
|
|||
public String getNextConstraintEmailValue(CassandraToSqlColumn column) { |
|||
int counter = this.nextContraintCounter(); |
|||
String[] emailValues = this.originalValue.split("@"); |
|||
String newValue = emailValues[0] + "+" + counter + "@" + emailValues[1]; |
|||
int overflow = newValue.length() - column.getSize(); |
|||
if (overflow > 0) { |
|||
newValue = emailValues[0].substring(0, emailValues[0].length()-overflow) + "+" + counter + "@" + emailValues[1]; |
|||
} |
|||
return newValue; |
|||
} |
|||
|
|||
public String getLogValue() { |
|||
if (this.value != null && this.value.length() > 255) { |
|||
return this.value.substring(0, 255) + "...[truncated " + (this.value.length() - 255) + " symbols]"; |
|||
} |
|||
return this.value; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,28 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
public enum CassandraToSqlColumnType { |
|||
ID, |
|||
DOUBLE, |
|||
INTEGER, |
|||
FLOAT, |
|||
BIGINT, |
|||
BOOLEAN, |
|||
STRING, |
|||
JSON, |
|||
ENUM_TO_INT |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import com.datastax.oss.driver.api.core.cql.Row; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; |
|||
|
|||
public class CassandraToSqlEventTsColumn extends CassandraToSqlColumn { |
|||
|
|||
CassandraToSqlEventTsColumn() { |
|||
super("id", "ts", CassandraToSqlColumnType.BIGINT, null, false); |
|||
} |
|||
|
|||
@Override |
|||
public String getColumnValue(Row row) { |
|||
UUID id = row.getUuid(getIndex()); |
|||
long ts = getTs(id); |
|||
return ts + ""; |
|||
} |
|||
|
|||
private long getTs(UUID uuid) { |
|||
return (uuid.timestamp() - EPOCH_DIFF) / 10000; |
|||
} |
|||
} |
|||
@ -0,0 +1,304 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import com.datastax.oss.driver.api.core.cql.ResultSet; |
|||
import com.datastax.oss.driver.api.core.cql.Row; |
|||
import com.datastax.oss.driver.api.core.cql.SimpleStatement; |
|||
import com.datastax.oss.driver.api.core.cql.Statement; |
|||
import lombok.Data; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.hibernate.internal.util.JdbcExceptionHelper; |
|||
import org.postgresql.util.PSQLException; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
import org.thingsboard.server.dao.cassandra.guava.GuavaSession; |
|||
|
|||
import java.sql.Connection; |
|||
import java.sql.DatabaseMetaData; |
|||
import java.sql.PreparedStatement; |
|||
import java.sql.SQLException; |
|||
import java.util.ArrayList; |
|||
import java.util.Arrays; |
|||
import java.util.Iterator; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
|
|||
@Data |
|||
@Slf4j |
|||
public class CassandraToSqlTable { |
|||
|
|||
private static final int DEFAULT_BATCH_SIZE = 10000; |
|||
|
|||
private String cassandraCf; |
|||
private String sqlTableName; |
|||
|
|||
private List<CassandraToSqlColumn> columns; |
|||
|
|||
private int batchSize = DEFAULT_BATCH_SIZE; |
|||
|
|||
private PreparedStatement sqlInsertStatement; |
|||
|
|||
public CassandraToSqlTable(String tableName, CassandraToSqlColumn... columns) { |
|||
this(tableName, tableName, DEFAULT_BATCH_SIZE, columns); |
|||
} |
|||
|
|||
public CassandraToSqlTable(String tableName, String sqlTableName, CassandraToSqlColumn... columns) { |
|||
this(tableName, sqlTableName, DEFAULT_BATCH_SIZE, columns); |
|||
} |
|||
|
|||
public CassandraToSqlTable(String tableName, int batchSize, CassandraToSqlColumn... columns) { |
|||
this(tableName, tableName, batchSize, columns); |
|||
} |
|||
|
|||
public CassandraToSqlTable(String cassandraCf, String sqlTableName, int batchSize, CassandraToSqlColumn... columns) { |
|||
this.cassandraCf = cassandraCf; |
|||
this.sqlTableName = sqlTableName; |
|||
this.batchSize = batchSize; |
|||
this.columns = Arrays.asList(columns); |
|||
for (int i=0;i<columns.length;i++) { |
|||
this.columns.get(i).setIndex(i); |
|||
this.columns.get(i).setSqlIndex(i+1); |
|||
} |
|||
} |
|||
|
|||
public void migrateToSql(GuavaSession session, Connection conn) throws SQLException { |
|||
log.info("[{}] Migrating data from cassandra '{}' Column Family to '{}' SQL table...", this.sqlTableName, this.cassandraCf, this.sqlTableName); |
|||
DatabaseMetaData metadata = conn.getMetaData(); |
|||
java.sql.ResultSet resultSet = metadata.getColumns(null, null, this.sqlTableName, null); |
|||
while (resultSet.next()) { |
|||
String name = resultSet.getString("COLUMN_NAME"); |
|||
int sqlType = resultSet.getInt("DATA_TYPE"); |
|||
int size = resultSet.getInt("COLUMN_SIZE"); |
|||
CassandraToSqlColumn column = this.getColumn(name); |
|||
column.setSize(size); |
|||
column.setSqlType(sqlType); |
|||
} |
|||
this.sqlInsertStatement = createSqlInsertStatement(conn); |
|||
Statement cassandraSelectStatement = createCassandraSelectStatement(); |
|||
cassandraSelectStatement.setPageSize(100); |
|||
ResultSet rs = session.execute(cassandraSelectStatement); |
|||
Iterator<Row> iter = rs.iterator(); |
|||
int rowCounter = 0; |
|||
List<CassandraToSqlColumnData[]> batchData; |
|||
boolean hasNext; |
|||
do { |
|||
batchData = this.extractBatchData(iter); |
|||
hasNext = batchData.size() == this.batchSize; |
|||
this.batchInsert(batchData, conn); |
|||
rowCounter += batchData.size(); |
|||
log.info("[{}] {} records migrated so far...", this.sqlTableName, rowCounter); |
|||
} while (hasNext); |
|||
this.sqlInsertStatement.close(); |
|||
log.info("[{}] {} total records migrated.", this.sqlTableName, rowCounter); |
|||
log.info("[{}] Finished migration data from cassandra '{}' Column Family to '{}' SQL table.", |
|||
this.sqlTableName, this.cassandraCf, this.sqlTableName); |
|||
} |
|||
|
|||
private List<CassandraToSqlColumnData[]> extractBatchData(Iterator<Row> iter) { |
|||
List<CassandraToSqlColumnData[]> batchData = new ArrayList<>(); |
|||
while (iter.hasNext() && batchData.size() < this.batchSize) { |
|||
Row row = iter.next(); |
|||
if (row != null) { |
|||
CassandraToSqlColumnData[] data = this.extractRowData(row); |
|||
batchData.add(data); |
|||
} |
|||
} |
|||
return batchData; |
|||
} |
|||
|
|||
private CassandraToSqlColumnData[] extractRowData(Row row) { |
|||
CassandraToSqlColumnData[] data = new CassandraToSqlColumnData[this.columns.size()]; |
|||
for (CassandraToSqlColumn column: this.columns) { |
|||
String value = column.getColumnValue(row); |
|||
data[column.getIndex()] = new CassandraToSqlColumnData(value); |
|||
} |
|||
return this.validateColumnData(data); |
|||
} |
|||
|
|||
protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { |
|||
for (int i=0;i<data.length;i++) { |
|||
CassandraToSqlColumn column = this.columns.get(i); |
|||
if (column.getType() == CassandraToSqlColumnType.STRING) { |
|||
CassandraToSqlColumnData columnData = data[i]; |
|||
String value = columnData.getValue(); |
|||
if (value != null && value.length() > column.getSize()) { |
|||
log.warn("[{}] Value size [{}] exceeds maximum size [{}] of column [{}] and will be truncated!", |
|||
this.sqlTableName, |
|||
value.length(), column.getSize(), column.getSqlColumnName()); |
|||
log.warn("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
value = value.substring(0, column.getSize()); |
|||
columnData.setOriginalValue(value); |
|||
columnData.setValue(value); |
|||
} |
|||
} |
|||
} |
|||
return data; |
|||
} |
|||
|
|||
protected void batchInsert(List<CassandraToSqlColumnData[]> batchData, Connection conn) throws SQLException { |
|||
boolean retry = false; |
|||
for (CassandraToSqlColumnData[] data : batchData) { |
|||
for (CassandraToSqlColumn column: this.columns) { |
|||
column.setColumnValue(this.sqlInsertStatement, data[column.getIndex()].getValue()); |
|||
} |
|||
try { |
|||
this.sqlInsertStatement.executeUpdate(); |
|||
} catch (SQLException e) { |
|||
if (this.handleInsertException(batchData, data, conn, e)) { |
|||
retry = true; |
|||
break; |
|||
} else { |
|||
throw e; |
|||
} |
|||
} |
|||
} |
|||
if (retry) { |
|||
this.batchInsert(batchData, conn); |
|||
} else { |
|||
conn.commit(); |
|||
} |
|||
} |
|||
|
|||
private boolean handleInsertException(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, |
|||
Connection conn, SQLException ex) throws SQLException { |
|||
conn.commit(); |
|||
String constraint = extractConstraintName(ex).orElse(null); |
|||
if (constraint != null) { |
|||
if (this.onConstraintViolation(batchData, data, constraint)) { |
|||
return true; |
|||
} else { |
|||
log.error("[{}] Unhandled constraint violation [{}] during insert!", this.sqlTableName, constraint); |
|||
log.error("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
} |
|||
} else { |
|||
log.error("[{}] Unhandled exception during insert!", this.sqlTableName); |
|||
log.error("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
private String dataToString(CassandraToSqlColumnData[] data) { |
|||
StringBuffer stringData = new StringBuffer("{\n"); |
|||
for (int i=0;i<data.length;i++) { |
|||
String columnName = this.columns.get(i).getSqlColumnName(); |
|||
String value = data[i].getLogValue(); |
|||
stringData.append("\"").append(columnName).append("\": ").append("[").append(value).append("]\n"); |
|||
} |
|||
stringData.append("}"); |
|||
return stringData.toString(); |
|||
} |
|||
|
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
return false; |
|||
} |
|||
|
|||
protected void handleUniqueNameViolation(CassandraToSqlColumnData[] data, String entityType) { |
|||
CassandraToSqlColumn nameColumn = this.getColumn("name"); |
|||
CassandraToSqlColumn searchTextColumn = this.getColumn("search_text"); |
|||
CassandraToSqlColumnData nameColumnData = data[nameColumn.getIndex()]; |
|||
CassandraToSqlColumnData searchTextColumnData = data[searchTextColumn.getIndex()]; |
|||
String prevName = nameColumnData.getValue(); |
|||
String newName = nameColumnData.getNextConstraintStringValue(nameColumn); |
|||
nameColumnData.setValue(newName); |
|||
searchTextColumnData.setValue(searchTextColumnData.getNextConstraintStringValue(searchTextColumn)); |
|||
String id = UUIDConverter.fromString(this.getColumnData(data, "id").getValue()).toString(); |
|||
log.warn("Found {} with duplicate name [id:[{}]]. Attempting to rename {} from '{}' to '{}'...", entityType, id, entityType, prevName, newName); |
|||
} |
|||
|
|||
protected void handleUniqueEmailViolation(CassandraToSqlColumnData[] data) { |
|||
CassandraToSqlColumn emailColumn = this.getColumn("email"); |
|||
CassandraToSqlColumn searchTextColumn = this.getColumn("search_text"); |
|||
CassandraToSqlColumnData emailColumnData = data[emailColumn.getIndex()]; |
|||
CassandraToSqlColumnData searchTextColumnData = data[searchTextColumn.getIndex()]; |
|||
String prevEmail = emailColumnData.getValue(); |
|||
String newEmail = emailColumnData.getNextConstraintEmailValue(emailColumn); |
|||
emailColumnData.setValue(newEmail); |
|||
searchTextColumnData.setValue(searchTextColumnData.getNextConstraintEmailValue(searchTextColumn)); |
|||
String id = UUIDConverter.fromString(this.getColumnData(data, "id").getValue()).toString(); |
|||
log.warn("Found user with duplicate email [id:[{}]]. Attempting to rename email from '{}' to '{}'...", id, prevEmail, newEmail); |
|||
} |
|||
|
|||
protected void ignoreRecord(List<CassandraToSqlColumnData[]> batchData, CassandraToSqlColumnData[] data) { |
|||
log.warn("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
int index = batchData.indexOf(data); |
|||
if (index > 0) { |
|||
batchData.remove(index); |
|||
} |
|||
} |
|||
|
|||
protected CassandraToSqlColumn getColumn(String sqlColumnName) { |
|||
return this.columns.stream().filter(col -> col.getSqlColumnName().equals(sqlColumnName)).findFirst().get(); |
|||
} |
|||
|
|||
protected CassandraToSqlColumnData getColumnData(CassandraToSqlColumnData[] data, String sqlColumnName) { |
|||
CassandraToSqlColumn column = this.getColumn(sqlColumnName); |
|||
return data[column.getIndex()]; |
|||
} |
|||
|
|||
private Optional<String> extractConstraintName(SQLException ex) { |
|||
final String sqlState = JdbcExceptionHelper.extractSqlState( ex ); |
|||
if (sqlState != null) { |
|||
String sqlStateClassCode = JdbcExceptionHelper.determineSqlStateClassCode( sqlState ); |
|||
if ( sqlStateClassCode != null ) { |
|||
if (Arrays.asList( |
|||
"23", // "integrity constraint violation"
|
|||
"27", // "triggered data change violation"
|
|||
"44" // "with check option violation"
|
|||
).contains(sqlStateClassCode)) { |
|||
if (ex instanceof PSQLException) { |
|||
return Optional.of(((PSQLException)ex).getServerErrorMessage().getConstraint()); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
return Optional.empty(); |
|||
} |
|||
|
|||
protected Statement createCassandraSelectStatement() { |
|||
StringBuilder selectStatementBuilder = new StringBuilder(); |
|||
selectStatementBuilder.append("SELECT "); |
|||
for (CassandraToSqlColumn column : columns) { |
|||
selectStatementBuilder.append(column.getCassandraColumnName()).append(","); |
|||
} |
|||
selectStatementBuilder.deleteCharAt(selectStatementBuilder.length() - 1); |
|||
selectStatementBuilder.append(" FROM ").append(cassandraCf); |
|||
return SimpleStatement.newInstance(selectStatementBuilder.toString()); |
|||
} |
|||
|
|||
private PreparedStatement createSqlInsertStatement(Connection conn) throws SQLException { |
|||
StringBuilder insertStatementBuilder = new StringBuilder(); |
|||
insertStatementBuilder.append("INSERT INTO ").append(this.sqlTableName).append(" ("); |
|||
for (CassandraToSqlColumn column : columns) { |
|||
insertStatementBuilder.append(column.getSqlColumnName()).append(","); |
|||
} |
|||
insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); |
|||
insertStatementBuilder.append(") VALUES ("); |
|||
for (CassandraToSqlColumn column : columns) { |
|||
if (column.getType() == CassandraToSqlColumnType.JSON) { |
|||
insertStatementBuilder.append("cast(? AS json)"); |
|||
} else { |
|||
insertStatementBuilder.append("?"); |
|||
} |
|||
insertStatementBuilder.append(","); |
|||
} |
|||
insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); |
|||
insertStatementBuilder.append(")"); |
|||
return conn.prepareStatement(insertStatementBuilder.toString()); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,222 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.hibernate.exception.ConstraintViolationException; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
import org.thingsboard.server.dao.cassandra.CassandraCluster; |
|||
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; |
|||
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; |
|||
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; |
|||
import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; |
|||
import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; |
|||
import org.thingsboard.server.dao.util.NoSqlTsDao; |
|||
import org.thingsboard.server.dao.util.SqlTsLatestDao; |
|||
import org.thingsboard.server.service.install.EntityDatabaseSchemaService; |
|||
|
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
import java.util.Arrays; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.bigintColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.booleanColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.doubleColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.idColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.jsonColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.stringColumn; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@NoSqlTsDao |
|||
@SqlTsLatestDao |
|||
@Slf4j |
|||
public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateService { |
|||
|
|||
private static final int MAX_KEY_LENGTH = 255; |
|||
private static final int MAX_STR_V_LENGTH = 10000000; |
|||
|
|||
@Autowired |
|||
private EntityDatabaseSchemaService entityDatabaseSchemaService; |
|||
|
|||
@Autowired |
|||
private InsertLatestTsRepository insertLatestTsRepository; |
|||
|
|||
@Autowired |
|||
protected CassandraCluster cluster; |
|||
|
|||
@Autowired |
|||
protected TsKvDictionaryRepository dictionaryRepository; |
|||
|
|||
@Value("${spring.datasource.url}") |
|||
protected String dbUrl; |
|||
|
|||
@Value("${spring.datasource.username}") |
|||
protected String dbUserName; |
|||
|
|||
@Value("${spring.datasource.password}") |
|||
protected String dbPassword; |
|||
|
|||
private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>(); |
|||
|
|||
protected static final ReentrantLock tsCreationLock = new ReentrantLock(); |
|||
|
|||
@Override |
|||
public void migrate() throws Exception { |
|||
log.info("Performing migration of latest timeseries data from cassandra to SQL database ..."); |
|||
entityDatabaseSchemaService.createDatabaseSchema(false); |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
conn.setAutoCommit(false); |
|||
for (CassandraToSqlTable table : tables) { |
|||
table.migrateToSql(cluster.getSession(), conn); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Unexpected error during ThingsBoard entities data migration!", e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
private List<CassandraToSqlTable> tables = Arrays.asList( |
|||
new CassandraToSqlTable("ts_kv_latest_cf", "ts_kv_latest", |
|||
idColumn("entity_id"), |
|||
stringColumn("key"), |
|||
bigintColumn("ts"), |
|||
booleanColumn("bool_v"), |
|||
stringColumn("str_v"), |
|||
bigintColumn("long_v"), |
|||
doubleColumn("dbl_v"), |
|||
jsonColumn("json_v")) { |
|||
|
|||
@Override |
|||
protected void batchInsert(List<CassandraToSqlColumnData[]> batchData, Connection conn) { |
|||
insertLatestTsRepository |
|||
.saveOrUpdate(batchData.stream().map(data -> getTsKvLatestEntity(data)).collect(Collectors.toList())); |
|||
} |
|||
|
|||
@Override |
|||
protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { |
|||
return data; |
|||
} |
|||
}); |
|||
|
|||
private TsKvLatestEntity getTsKvLatestEntity(CassandraToSqlColumnData[] data) { |
|||
TsKvLatestEntity latestEntity = new TsKvLatestEntity(); |
|||
latestEntity.setEntityId(UUIDConverter.fromString(data[0].getValue())); |
|||
latestEntity.setKey(getOrSaveKeyId(data[1].getValue())); |
|||
latestEntity.setTs(Long.parseLong(data[2].getValue())); |
|||
|
|||
String strV = data[4].getValue(); |
|||
if (strV != null) { |
|||
if (strV.length() > MAX_STR_V_LENGTH) { |
|||
log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [str_v] and will be truncated!", |
|||
strV.length(), MAX_STR_V_LENGTH); |
|||
log.warn("Affected data:\n{}", strV); |
|||
strV = strV.substring(0, MAX_STR_V_LENGTH); |
|||
} |
|||
latestEntity.setStrValue(strV); |
|||
} else { |
|||
Long longV = null; |
|||
try { |
|||
longV = Long.parseLong(data[5].getValue()); |
|||
} catch (Exception e) { |
|||
} |
|||
if (longV != null) { |
|||
latestEntity.setLongValue(longV); |
|||
} else { |
|||
Double doubleV = null; |
|||
try { |
|||
doubleV = Double.parseDouble(data[6].getValue()); |
|||
} catch (Exception e) { |
|||
} |
|||
if (doubleV != null) { |
|||
latestEntity.setDoubleValue(doubleV); |
|||
} else { |
|||
|
|||
String jsonV = data[7].getValue(); |
|||
if (StringUtils.isNoneEmpty(jsonV)) { |
|||
latestEntity.setJsonValue(jsonV); |
|||
} else { |
|||
Boolean boolV = null; |
|||
try { |
|||
boolV = Boolean.parseBoolean(data[3].getValue()); |
|||
} catch (Exception e) { |
|||
} |
|||
if (boolV != null) { |
|||
latestEntity.setBooleanValue(boolV); |
|||
} else { |
|||
log.warn("All values in key-value row are nullable "); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
return latestEntity; |
|||
} |
|||
|
|||
protected Integer getOrSaveKeyId(String strKey) { |
|||
if (strKey.length() > MAX_KEY_LENGTH) { |
|||
log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [key] and will be truncated!", |
|||
strKey.length(), MAX_KEY_LENGTH); |
|||
log.warn("Affected data:\n{}", strKey); |
|||
strKey = strKey.substring(0, MAX_KEY_LENGTH); |
|||
} |
|||
|
|||
Integer keyId = tsKvDictionaryMap.get(strKey); |
|||
if (keyId == null) { |
|||
Optional<TsKvDictionary> tsKvDictionaryOptional; |
|||
tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); |
|||
if (!tsKvDictionaryOptional.isPresent()) { |
|||
tsCreationLock.lock(); |
|||
try { |
|||
tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); |
|||
if (!tsKvDictionaryOptional.isPresent()) { |
|||
TsKvDictionary tsKvDictionary = new TsKvDictionary(); |
|||
tsKvDictionary.setKey(strKey); |
|||
try { |
|||
TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); |
|||
tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); |
|||
keyId = saved.getKeyId(); |
|||
} catch (ConstraintViolationException e) { |
|||
tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); |
|||
TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); |
|||
tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); |
|||
keyId = dictionary.getKeyId(); |
|||
} |
|||
} else { |
|||
keyId = tsKvDictionaryOptional.get().getKeyId(); |
|||
} |
|||
} finally { |
|||
tsCreationLock.unlock(); |
|||
} |
|||
} else { |
|||
keyId = tsKvDictionaryOptional.get().getKeyId(); |
|||
tsKvDictionaryMap.put(strKey, keyId); |
|||
} |
|||
} |
|||
return keyId; |
|||
} |
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
public interface EntitiesMigrateService { |
|||
|
|||
void migrate() throws Exception; |
|||
|
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
public interface TsLatestMigrateService { |
|||
|
|||
void migrate() throws Exception; |
|||
} |
|||
@ -0,0 +1,103 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.query; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AlarmData; |
|||
import org.thingsboard.server.common.data.query.AlarmDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityCountQuery; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataPageLink; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityDataSortOrder; |
|||
import org.thingsboard.server.common.data.query.EntityKey; |
|||
import org.thingsboard.server.common.data.query.EntityKeyType; |
|||
import org.thingsboard.server.dao.alarm.AlarmService; |
|||
import org.thingsboard.server.dao.entity.EntityService; |
|||
import org.thingsboard.server.dao.model.ModelConstants; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
|
|||
import java.util.LinkedHashMap; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
@TbCoreComponent |
|||
public class DefaultEntityQueryService implements EntityQueryService { |
|||
|
|||
@Autowired |
|||
private EntityService entityService; |
|||
|
|||
@Autowired |
|||
private AlarmService alarmService; |
|||
|
|||
@Value("${server.ws.max_entities_per_alarm_subscription:1000}") |
|||
private int maxEntitiesPerAlarmSubscription; |
|||
|
|||
@Override |
|||
public long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query) { |
|||
return entityService.countEntitiesByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query); |
|||
} |
|||
|
|||
@Override |
|||
public PageData<EntityData> findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query) { |
|||
return entityService.findEntityDataByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query); |
|||
} |
|||
|
|||
@Override |
|||
public PageData<AlarmData> findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query) { |
|||
EntityDataQuery entityDataQuery = this.buildEntityDataQuery(query); |
|||
PageData<EntityData> entities = entityService.findEntityDataByQuery(securityUser.getTenantId(), |
|||
securityUser.getCustomerId(), entityDataQuery); |
|||
if (entities.getTotalElements() > 0) { |
|||
LinkedHashMap<EntityId, EntityData> entitiesMap = new LinkedHashMap<>(); |
|||
for (EntityData entityData : entities.getData()) { |
|||
entitiesMap.put(entityData.getEntityId(), entityData); |
|||
} |
|||
PageData<AlarmData> alarms = alarmService.findAlarmDataByQueryForEntities(securityUser.getTenantId(), |
|||
securityUser.getCustomerId(), query, entitiesMap.keySet()); |
|||
for (AlarmData alarmData : alarms.getData()) { |
|||
EntityId entityId = alarmData.getEntityId(); |
|||
if (entityId != null) { |
|||
EntityData entityData = entitiesMap.get(entityId); |
|||
if (entityData != null) { |
|||
alarmData.getLatest().putAll(entityData.getLatest()); |
|||
} |
|||
} |
|||
} |
|||
return alarms; |
|||
} else { |
|||
return new PageData<>(); |
|||
} |
|||
} |
|||
|
|||
private EntityDataQuery buildEntityDataQuery(AlarmDataQuery query) { |
|||
EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); |
|||
EntityDataSortOrder entitiesSortOrder; |
|||
if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { |
|||
entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); |
|||
} else { |
|||
entitiesSortOrder = sortOrder; |
|||
} |
|||
EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, entitiesSortOrder); |
|||
return new EntityDataQuery(query.getEntityFilter(), edpl, query.getEntityFields(), query.getLatestValues(), query.getKeyFilters()); |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.query; |
|||
|
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AlarmData; |
|||
import org.thingsboard.server.common.data.query.AlarmDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityCountQuery; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
|
|||
public interface EntityQueryService { |
|||
|
|||
long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query); |
|||
|
|||
PageData<EntityData> findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query); |
|||
|
|||
PageData<AlarmData> findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query); |
|||
|
|||
} |
|||
@ -0,0 +1,484 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.subscription; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.Getter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.checkerframework.checker.nullness.qual.Nullable; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Lazy; |
|||
import org.springframework.scheduling.annotation.Scheduled; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.id.CustomerId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AlarmDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataPageLink; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityDataSortOrder; |
|||
import org.thingsboard.server.common.data.query.EntityKey; |
|||
import org.thingsboard.server.common.data.query.EntityKeyType; |
|||
import org.thingsboard.server.common.data.query.TsValue; |
|||
import org.thingsboard.server.dao.alarm.AlarmService; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.entity.EntityService; |
|||
import org.thingsboard.server.dao.model.ModelConstants; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.executors.DbCallbackExecutorService; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.AlarmDataCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.AlarmDataUpdate; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.EntityHistoryCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.GetTsCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.LatestValueCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.TimeSeriesCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.UnsubscribeCmd; |
|||
import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode; |
|||
|
|||
import javax.annotation.PostConstruct; |
|||
import javax.annotation.PreDestroy; |
|||
import java.util.ArrayList; |
|||
import java.util.Arrays; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.LinkedHashMap; |
|||
import java.util.LinkedHashSet; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ExecutionException; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.ScheduledExecutorService; |
|||
import java.util.concurrent.ScheduledFuture; |
|||
import java.util.concurrent.ThreadFactory; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
import java.util.concurrent.atomic.AtomicLong; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Slf4j |
|||
@TbCoreComponent |
|||
@Service |
|||
public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubscriptionService { |
|||
|
|||
private static final int DEFAULT_LIMIT = 100; |
|||
private final Map<String, Map<Integer, TbAbstractDataSubCtx>> subscriptionsBySessionId = new ConcurrentHashMap<>(); |
|||
|
|||
@Autowired |
|||
private TelemetryWebSocketService wsService; |
|||
|
|||
@Autowired |
|||
private EntityService entityService; |
|||
|
|||
@Autowired |
|||
private AlarmService alarmService; |
|||
|
|||
@Autowired |
|||
private AttributesService attributesService; |
|||
|
|||
@Autowired |
|||
@Lazy |
|||
private TbLocalSubscriptionService localSubscriptionService; |
|||
|
|||
@Autowired |
|||
private TimeseriesService tsService; |
|||
|
|||
@Autowired |
|||
private TbServiceInfoProvider serviceInfoProvider; |
|||
|
|||
@Autowired |
|||
@Getter |
|||
private DbCallbackExecutorService dbCallbackExecutor; |
|||
|
|||
private ScheduledExecutorService scheduler; |
|||
|
|||
@Value("${database.ts.type}") |
|||
private String databaseTsType; |
|||
@Value("${server.ws.dynamic_page_link.refresh_interval:6}") |
|||
private long dynamicPageLinkRefreshInterval; |
|||
@Value("${server.ws.dynamic_page_link.refresh_pool_size:1}") |
|||
private int dynamicPageLinkRefreshPoolSize; |
|||
@Value("${server.ws.max_entities_per_data_subscription:1000}") |
|||
private int maxEntitiesPerDataSubscription; |
|||
@Value("${server.ws.max_entities_per_alarm_subscription:1000}") |
|||
private int maxEntitiesPerAlarmSubscription; |
|||
|
|||
private ExecutorService wsCallBackExecutor; |
|||
private boolean tsInSqlDB; |
|||
private String serviceId; |
|||
private SubscriptionServiceStatistics stats = new SubscriptionServiceStatistics(); |
|||
|
|||
@PostConstruct |
|||
public void initExecutor() { |
|||
serviceId = serviceInfoProvider.getServiceId(); |
|||
wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("ws-entity-sub-callback")); |
|||
tsInSqlDB = databaseTsType.equalsIgnoreCase("sql") || databaseTsType.equalsIgnoreCase("timescale"); |
|||
ThreadFactory tbThreadFactory = ThingsBoardThreadFactory.forName("ws-entity-sub-scheduler"); |
|||
if (dynamicPageLinkRefreshPoolSize == 1) { |
|||
scheduler = Executors.newSingleThreadScheduledExecutor(tbThreadFactory); |
|||
} else { |
|||
scheduler = Executors.newScheduledThreadPool(dynamicPageLinkRefreshPoolSize, tbThreadFactory); |
|||
} |
|||
} |
|||
|
|||
@PreDestroy |
|||
public void shutdownExecutor() { |
|||
if (wsCallBackExecutor != null) { |
|||
wsCallBackExecutor.shutdownNow(); |
|||
} |
|||
if (scheduler != null) { |
|||
scheduler.shutdownNow(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void handleCmd(TelemetryWebSocketSessionRef session, EntityDataCmd cmd) { |
|||
TbEntityDataSubCtx ctx = getSubCtx(session.getSessionId(), cmd.getCmdId()); |
|||
if (ctx != null) { |
|||
log.debug("[{}][{}] Updating existing subscriptions using: {}", session.getSessionId(), cmd.getCmdId(), cmd); |
|||
if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null || cmd.getHistoryCmd() != null) { |
|||
ctx.clearEntitySubscriptions(); |
|||
} |
|||
} else { |
|||
log.debug("[{}][{}] Creating new subscription using: {}", session.getSessionId(), cmd.getCmdId(), cmd); |
|||
ctx = createSubCtx(session, cmd); |
|||
} |
|||
ctx.setCurrentCmd(cmd); |
|||
if (cmd.getQuery() != null) { |
|||
if (ctx.getQuery() == null) { |
|||
log.debug("[{}][{}] Initializing data using query: {}", session.getSessionId(), cmd.getCmdId(), cmd.getQuery()); |
|||
} else { |
|||
log.debug("[{}][{}] Updating data using query: {}", session.getSessionId(), cmd.getCmdId(), cmd.getQuery()); |
|||
} |
|||
ctx.setAndResolveQuery(cmd.getQuery()); |
|||
TenantId tenantId = ctx.getTenantId(); |
|||
CustomerId customerId = ctx.getCustomerId(); |
|||
EntityDataQuery query = ctx.getQuery(); |
|||
//Step 1. Update existing query with the contents of LatestValueCmd
|
|||
if (cmd.getLatestCmd() != null) { |
|||
cmd.getLatestCmd().getKeys().forEach(key -> { |
|||
if (!query.getLatestValues().contains(key)) { |
|||
query.getLatestValues().add(key); |
|||
} |
|||
}); |
|||
} |
|||
long start = System.currentTimeMillis(); |
|||
ctx.fetchData(); |
|||
long end = System.currentTimeMillis(); |
|||
stats.getRegularQueryInvocationCnt().incrementAndGet(); |
|||
stats.getRegularQueryTimeSpent().addAndGet(end - start); |
|||
ctx.cancelTasks(); |
|||
if (ctx.getQuery().getPageLink().isDynamic()) { |
|||
//TODO: validate number of dynamic page links against rate limits. Ignore dynamic flag if limit is reached.
|
|||
TbEntityDataSubCtx finalCtx = ctx; |
|||
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay( |
|||
() -> refreshDynamicQuery(tenantId, customerId, finalCtx), |
|||
dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS); |
|||
finalCtx.setRefreshTask(task); |
|||
} |
|||
} |
|||
ListenableFuture<TbEntityDataSubCtx> historyFuture; |
|||
if (cmd.getHistoryCmd() != null) { |
|||
log.trace("[{}][{}] Going to process history command: {}", session.getSessionId(), cmd.getCmdId(), cmd.getHistoryCmd()); |
|||
historyFuture = handleHistoryCmd(ctx, cmd.getHistoryCmd()); |
|||
} else { |
|||
historyFuture = Futures.immediateFuture(ctx); |
|||
} |
|||
Futures.addCallback(historyFuture, new FutureCallback<TbEntityDataSubCtx>() { |
|||
@Override |
|||
public void onSuccess(@Nullable TbEntityDataSubCtx theCtx) { |
|||
if (cmd.getLatestCmd() != null) { |
|||
handleLatestCmd(theCtx, cmd.getLatestCmd()); |
|||
} else if (cmd.getTsCmd() != null) { |
|||
handleTimeSeriesCmd(theCtx, cmd.getTsCmd()); |
|||
} else if (!theCtx.isInitialDataSent()) { |
|||
EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription()); |
|||
wsService.sendWsMsg(theCtx.getSessionId(), update); |
|||
theCtx.setInitialDataSent(true); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.warn("[{}][{}] Failed to process command", session.getSessionId(), cmd.getCmdId()); |
|||
} |
|||
}, wsCallBackExecutor); |
|||
} |
|||
|
|||
@Override |
|||
public void handleCmd(TelemetryWebSocketSessionRef session, AlarmDataCmd cmd) { |
|||
TbAlarmDataSubCtx ctx = getSubCtx(session.getSessionId(), cmd.getCmdId()); |
|||
if (ctx == null) { |
|||
log.debug("[{}][{}] Creating new alarm subscription using: {}", session.getSessionId(), cmd.getCmdId(), cmd); |
|||
ctx = createSubCtx(session, cmd); |
|||
} |
|||
ctx.setAndResolveQuery(cmd.getQuery()); |
|||
AlarmDataQuery adq = ctx.getQuery(); |
|||
long start = System.currentTimeMillis(); |
|||
ctx.fetchData(); |
|||
long end = System.currentTimeMillis(); |
|||
stats.getRegularQueryInvocationCnt().incrementAndGet(); |
|||
stats.getRegularQueryTimeSpent().addAndGet(end - start); |
|||
List<EntityData> entities = ctx.getEntitiesData(); |
|||
ctx.cancelTasks(); |
|||
ctx.clearEntitySubscriptions(); |
|||
if (entities.isEmpty()) { |
|||
AlarmDataUpdate update = new AlarmDataUpdate(cmd.getCmdId(), new PageData<>(), null, 0, 0); |
|||
wsService.sendWsMsg(ctx.getSessionId(), update); |
|||
} else { |
|||
ctx.fetchAlarms(); |
|||
ctx.createSubscriptions(cmd.getQuery().getLatestValues(), true); |
|||
if (adq.getPageLink().getTimeWindow() > 0) { |
|||
TbAlarmDataSubCtx finalCtx = ctx; |
|||
ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay( |
|||
finalCtx::cleanupOldAlarms, dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS); |
|||
finalCtx.setRefreshTask(task); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void refreshDynamicQuery(TenantId tenantId, CustomerId customerId, TbEntityDataSubCtx finalCtx) { |
|||
try { |
|||
long start = System.currentTimeMillis(); |
|||
finalCtx.update(); |
|||
long end = System.currentTimeMillis(); |
|||
stats.getDynamicQueryInvocationCnt().incrementAndGet(); |
|||
stats.getDynamicQueryTimeSpent().addAndGet(end - start); |
|||
} catch (Exception e) { |
|||
log.warn("[{}][{}] Failed to refresh query", finalCtx.getSessionId(), finalCtx.getCmdId(), e); |
|||
} |
|||
} |
|||
|
|||
@Scheduled(fixedDelayString = "${server.ws.dynamic_page_link.stats:10000}") |
|||
public void printStats() { |
|||
int alarmQueryInvocationCntValue = stats.getAlarmQueryInvocationCnt().getAndSet(0); |
|||
long alarmQueryInvocationTimeValue = stats.getAlarmQueryTimeSpent().getAndSet(0); |
|||
int regularQueryInvocationCntValue = stats.getRegularQueryInvocationCnt().getAndSet(0); |
|||
long regularQueryInvocationTimeValue = stats.getRegularQueryTimeSpent().getAndSet(0); |
|||
int dynamicQueryInvocationCntValue = stats.getDynamicQueryInvocationCnt().getAndSet(0); |
|||
long dynamicQueryInvocationTimeValue = stats.getDynamicQueryTimeSpent().getAndSet(0); |
|||
long dynamicQueryCnt = subscriptionsBySessionId.values().stream().map(Map::values).count(); |
|||
if (regularQueryInvocationCntValue > 0 || dynamicQueryInvocationCntValue > 0 || dynamicQueryCnt > 0 || alarmQueryInvocationCntValue > 0) { |
|||
log.info("Stats: regularQueryInvocationCnt = [{}], regularQueryInvocationTime = [{}], " + |
|||
"dynamicQueryCnt = [{}] dynamicQueryInvocationCnt = [{}], dynamicQueryInvocationTime = [{}], " + |
|||
"alarmQueryInvocationCnt = [{}], alarmQueryInvocationTime = [{}]", |
|||
regularQueryInvocationCntValue, regularQueryInvocationTimeValue, |
|||
dynamicQueryCnt, dynamicQueryInvocationCntValue, dynamicQueryInvocationTimeValue, |
|||
alarmQueryInvocationCntValue, alarmQueryInvocationTimeValue); |
|||
} |
|||
} |
|||
|
|||
private TbEntityDataSubCtx createSubCtx(TelemetryWebSocketSessionRef sessionRef, EntityDataCmd cmd) { |
|||
Map<Integer, TbAbstractDataSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new HashMap<>()); |
|||
TbEntityDataSubCtx ctx = new TbEntityDataSubCtx(serviceId, wsService, entityService, localSubscriptionService, |
|||
attributesService, stats, sessionRef, cmd.getCmdId(), maxEntitiesPerDataSubscription); |
|||
ctx.setAndResolveQuery(cmd.getQuery()); |
|||
sessionSubs.put(cmd.getCmdId(), ctx); |
|||
return ctx; |
|||
} |
|||
|
|||
private TbAlarmDataSubCtx createSubCtx(TelemetryWebSocketSessionRef sessionRef, AlarmDataCmd cmd) { |
|||
Map<Integer, TbAbstractDataSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new HashMap<>()); |
|||
TbAlarmDataSubCtx ctx = new TbAlarmDataSubCtx(serviceId, wsService, entityService, localSubscriptionService, |
|||
attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription); |
|||
ctx.setAndResolveQuery(cmd.getQuery()); |
|||
sessionSubs.put(cmd.getCmdId(), ctx); |
|||
return ctx; |
|||
} |
|||
|
|||
private <T extends TbAbstractDataSubCtx> T getSubCtx(String sessionId, int cmdId) { |
|||
Map<Integer, TbAbstractDataSubCtx> sessionSubs = subscriptionsBySessionId.get(sessionId); |
|||
if (sessionSubs != null) { |
|||
return (T) sessionSubs.get(cmdId); |
|||
} else { |
|||
return null; |
|||
} |
|||
} |
|||
|
|||
private ListenableFuture<TbEntityDataSubCtx> handleTimeSeriesCmd(TbEntityDataSubCtx ctx, TimeSeriesCmd cmd) { |
|||
log.debug("[{}][{}] Fetching time-series data for last {} ms for keys: ({})", ctx.getSessionId(), ctx.getCmdId(), cmd.getTimeWindow(), cmd.getKeys()); |
|||
return handleGetTsCmd(ctx, cmd, true); |
|||
} |
|||
|
|||
|
|||
private ListenableFuture<TbEntityDataSubCtx> handleHistoryCmd(TbEntityDataSubCtx ctx, EntityHistoryCmd cmd) { |
|||
log.debug("[{}][{}] Fetching history data for start {} and end {} ms for keys: ({})", ctx.getSessionId(), ctx.getCmdId(), cmd.getStartTs(), cmd.getEndTs(), cmd.getKeys()); |
|||
return handleGetTsCmd(ctx, cmd, false); |
|||
} |
|||
|
|||
private ListenableFuture<TbEntityDataSubCtx> handleGetTsCmd(TbEntityDataSubCtx ctx, GetTsCmd cmd, boolean subscribe) { |
|||
List<String> keys = cmd.getKeys(); |
|||
List<ReadTsKvQuery> finalTsKvQueryList; |
|||
List<ReadTsKvQuery> tsKvQueryList = cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery( |
|||
key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg() |
|||
)).collect(Collectors.toList()); |
|||
if (cmd.isFetchLatestPreviousPoint()) { |
|||
finalTsKvQueryList = new ArrayList<>(tsKvQueryList); |
|||
finalTsKvQueryList.addAll(cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery( |
|||
key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg() |
|||
)).collect(Collectors.toList())); |
|||
} else { |
|||
finalTsKvQueryList = tsKvQueryList; |
|||
} |
|||
Map<EntityData, ListenableFuture<List<TsKvEntry>>> fetchResultMap = new HashMap<>(); |
|||
ctx.getData().getData().forEach(entityData -> fetchResultMap.put(entityData, |
|||
tsService.findAll(ctx.getTenantId(), entityData.getEntityId(), finalTsKvQueryList))); |
|||
return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> { |
|||
fetchResultMap.forEach((entityData, future) -> { |
|||
Map<String, List<TsValue>> keyData = new LinkedHashMap<>(); |
|||
cmd.getKeys().forEach(key -> keyData.put(key, new ArrayList<>())); |
|||
try { |
|||
List<TsKvEntry> entityTsData = future.get(); |
|||
if (entityTsData != null) { |
|||
entityTsData.forEach(entry -> keyData.get(entry.getKey()).add(new TsValue(entry.getTs(), entry.getValueAsString()))); |
|||
} |
|||
keyData.forEach((k, v) -> entityData.getTimeseries().put(k, v.toArray(new TsValue[v.size()]))); |
|||
if (cmd.isFetchLatestPreviousPoint()) { |
|||
entityData.getTimeseries().values().forEach(dataArray -> { |
|||
Arrays.sort(dataArray, (o1, o2) -> Long.compare(o2.getTs(), o1.getTs())); |
|||
}); |
|||
} |
|||
} catch (InterruptedException | ExecutionException e) { |
|||
log.warn("[{}][{}][{}] Failed to fetch historical data", ctx.getSessionId(), ctx.getCmdId(), entityData.getEntityId(), e); |
|||
wsService.sendWsMsg(ctx.getSessionId(), |
|||
new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to fetch historical data!")); |
|||
} |
|||
}); |
|||
EntityDataUpdate update; |
|||
if (!ctx.isInitialDataSent()) { |
|||
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription()); |
|||
ctx.setInitialDataSent(true); |
|||
} else { |
|||
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription()); |
|||
} |
|||
wsService.sendWsMsg(ctx.getSessionId(), update); |
|||
if (subscribe) { |
|||
ctx.createSubscriptions(keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).collect(Collectors.toList()), false); |
|||
} |
|||
ctx.getData().getData().forEach(ed -> ed.getTimeseries().clear()); |
|||
return ctx; |
|||
}, wsCallBackExecutor); |
|||
} |
|||
|
|||
private void handleLatestCmd(TbEntityDataSubCtx ctx, LatestValueCmd latestCmd) { |
|||
log.trace("[{}][{}] Going to process latest command: {}", ctx.getSessionId(), ctx.getCmdId(), latestCmd); |
|||
//Fetch the latest values for telemetry keys (in case they are not copied from NoSQL to SQL DB in hybrid mode.
|
|||
if (!tsInSqlDB) { |
|||
log.trace("[{}][{}] Going to fetch missing latest values: {}", ctx.getSessionId(), ctx.getCmdId(), latestCmd); |
|||
List<String> allTsKeys = latestCmd.getKeys().stream() |
|||
.filter(key -> key.getType().equals(EntityKeyType.TIME_SERIES)) |
|||
.map(EntityKey::getKey).collect(Collectors.toList()); |
|||
|
|||
Map<EntityData, ListenableFuture<Map<String, TsValue>>> missingTelemetryFutures = new HashMap<>(); |
|||
for (EntityData entityData : ctx.getData().getData()) { |
|||
Map<EntityKeyType, Map<String, TsValue>> latestEntityData = entityData.getLatest(); |
|||
Map<String, TsValue> tsEntityData = latestEntityData.get(EntityKeyType.TIME_SERIES); |
|||
Set<String> missingTsKeys = new LinkedHashSet<>(allTsKeys); |
|||
if (tsEntityData != null) { |
|||
missingTsKeys.removeAll(tsEntityData.keySet()); |
|||
} else { |
|||
tsEntityData = new HashMap<>(); |
|||
latestEntityData.put(EntityKeyType.TIME_SERIES, tsEntityData); |
|||
} |
|||
|
|||
ListenableFuture<List<TsKvEntry>> missingTsData = tsService.findLatest(ctx.getTenantId(), entityData.getEntityId(), missingTsKeys); |
|||
missingTelemetryFutures.put(entityData, Futures.transform(missingTsData, this::toTsValue, MoreExecutors.directExecutor())); |
|||
} |
|||
Futures.addCallback(Futures.allAsList(missingTelemetryFutures.values()), new FutureCallback<List<Map<String, TsValue>>>() { |
|||
@Override |
|||
public void onSuccess(@Nullable List<Map<String, TsValue>> result) { |
|||
missingTelemetryFutures.forEach((key, value) -> { |
|||
try { |
|||
key.getLatest().get(EntityKeyType.TIME_SERIES).putAll(value.get()); |
|||
} catch (InterruptedException | ExecutionException e) { |
|||
log.warn("[{}][{}] Failed to lookup latest telemetry: {}:{}", ctx.getSessionId(), ctx.getCmdId(), key.getEntityId(), allTsKeys, e); |
|||
} |
|||
}); |
|||
EntityDataUpdate update; |
|||
if (!ctx.isInitialDataSent()) { |
|||
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription()); |
|||
ctx.setInitialDataSent(true); |
|||
} else { |
|||
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription()); |
|||
} |
|||
wsService.sendWsMsg(ctx.getSessionId(), update); |
|||
ctx.createSubscriptions(latestCmd.getKeys(), true); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.warn("[{}][{}] Failed to process websocket command: {}:{}", ctx.getSessionId(), ctx.getCmdId(), ctx.getQuery(), latestCmd, t); |
|||
wsService.sendWsMsg(ctx.getSessionId(), |
|||
new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to process websocket command!")); |
|||
} |
|||
}, wsCallBackExecutor); |
|||
} else { |
|||
if (!ctx.isInitialDataSent()) { |
|||
EntityDataUpdate update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription()); |
|||
wsService.sendWsMsg(ctx.getSessionId(), update); |
|||
ctx.setInitialDataSent(true); |
|||
} |
|||
ctx.createSubscriptions(latestCmd.getKeys(), true); |
|||
} |
|||
} |
|||
|
|||
private Map<String, TsValue> toTsValue(List<TsKvEntry> data) { |
|||
return data.stream().collect(Collectors.toMap(TsKvEntry::getKey, value -> new TsValue(value.getTs(), value.getValueAsString()))); |
|||
} |
|||
|
|||
@Override |
|||
public void cancelSubscription(String sessionId, UnsubscribeCmd cmd) { |
|||
cleanupAndCancel(getSubCtx(sessionId, cmd.getCmdId())); |
|||
} |
|||
|
|||
private void cleanupAndCancel(TbAbstractDataSubCtx ctx) { |
|||
if (ctx != null) { |
|||
ctx.cancelTasks(); |
|||
ctx.clearEntitySubscriptions(); |
|||
ctx.clearDynamicValueSubscriptions(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void cancelAllSessionSubscriptions(String sessionId) { |
|||
Map<Integer, TbAbstractDataSubCtx> sessionSubs = subscriptionsBySessionId.remove(sessionId); |
|||
if (sessionSubs != null) { |
|||
sessionSubs.values().stream().filter(sub -> sub instanceof TbEntityDataSubCtx).map(sub -> (TbEntityDataSubCtx) sub).forEach(this::cleanupAndCancel); |
|||
} |
|||
} |
|||
|
|||
private int getLimit(int limit) { |
|||
return limit == 0 ? DEFAULT_LIMIT : limit; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.subscription; |
|||
|
|||
import lombok.Data; |
|||
|
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
import java.util.concurrent.atomic.AtomicLong; |
|||
|
|||
@Data |
|||
public class SubscriptionServiceStatistics { |
|||
private AtomicInteger alarmQueryInvocationCnt = new AtomicInteger(); |
|||
private AtomicInteger regularQueryInvocationCnt = new AtomicInteger(); |
|||
private AtomicInteger dynamicQueryInvocationCnt = new AtomicInteger(); |
|||
private AtomicLong alarmQueryTimeSpent = new AtomicLong(); |
|||
private AtomicLong regularQueryTimeSpent = new AtomicLong(); |
|||
private AtomicLong dynamicQueryTimeSpent = new AtomicLong(); |
|||
} |
|||
@ -0,0 +1,470 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.subscription; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.Data; |
|||
import lombok.Getter; |
|||
import lombok.Setter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.common.data.id.CustomerId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.UserId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AbstractDataQuery; |
|||
import org.thingsboard.server.common.data.query.ComplexFilterPredicate; |
|||
import org.thingsboard.server.common.data.query.DynamicValue; |
|||
import org.thingsboard.server.common.data.query.DynamicValueSourceType; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataPageLink; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityKey; |
|||
import org.thingsboard.server.common.data.query.EntityKeyType; |
|||
import org.thingsboard.server.common.data.query.FilterPredicateType; |
|||
import org.thingsboard.server.common.data.query.KeyFilter; |
|||
import org.thingsboard.server.common.data.query.KeyFilterPredicate; |
|||
import org.thingsboard.server.common.data.query.SimpleKeyFilterPredicate; |
|||
import org.thingsboard.server.common.data.query.TsValue; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.entity.EntityService; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef; |
|||
import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Arrays; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Optional; |
|||
import java.util.Set; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ExecutionException; |
|||
import java.util.concurrent.ScheduledFuture; |
|||
import java.util.function.Function; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Slf4j |
|||
@Data |
|||
public abstract class TbAbstractDataSubCtx<T extends AbstractDataQuery<? extends EntityDataPageLink>> { |
|||
|
|||
protected final String serviceId; |
|||
protected final SubscriptionServiceStatistics stats; |
|||
protected final TelemetryWebSocketService wsService; |
|||
protected final EntityService entityService; |
|||
protected final TbLocalSubscriptionService localSubscriptionService; |
|||
protected final AttributesService attributesService; |
|||
protected final TelemetryWebSocketSessionRef sessionRef; |
|||
protected final int cmdId; |
|||
protected final Map<Integer, EntityId> subToEntityIdMap; |
|||
protected final Set<Integer> subToDynamicValueKeySet; |
|||
@Getter |
|||
protected final Map<DynamicValueKey, List<DynamicValue>> dynamicValues; |
|||
@Getter |
|||
protected PageData<EntityData> data; |
|||
@Getter |
|||
@Setter |
|||
protected T query; |
|||
@Setter |
|||
protected volatile ScheduledFuture<?> refreshTask; |
|||
|
|||
public TbAbstractDataSubCtx(String serviceId, TelemetryWebSocketService wsService, |
|||
EntityService entityService, TbLocalSubscriptionService localSubscriptionService, |
|||
AttributesService attributesService, SubscriptionServiceStatistics stats, |
|||
TelemetryWebSocketSessionRef sessionRef, int cmdId) { |
|||
this.serviceId = serviceId; |
|||
this.wsService = wsService; |
|||
this.entityService = entityService; |
|||
this.localSubscriptionService = localSubscriptionService; |
|||
this.attributesService = attributesService; |
|||
this.stats = stats; |
|||
this.sessionRef = sessionRef; |
|||
this.cmdId = cmdId; |
|||
this.subToEntityIdMap = new ConcurrentHashMap<>(); |
|||
this.subToDynamicValueKeySet = ConcurrentHashMap.newKeySet(); |
|||
this.dynamicValues = new ConcurrentHashMap<>(); |
|||
} |
|||
|
|||
public void setAndResolveQuery(T query) { |
|||
dynamicValues.clear(); |
|||
this.query = query; |
|||
if (query.getKeyFilters() != null) { |
|||
for (KeyFilter filter : query.getKeyFilters()) { |
|||
registerDynamicValues(filter.getPredicate()); |
|||
} |
|||
} |
|||
resolve(getTenantId(), getCustomerId(), getUserId()); |
|||
} |
|||
|
|||
public void resolve(TenantId tenantId, CustomerId customerId, UserId userId) { |
|||
List<ListenableFuture<DynamicValueKeySub>> futures = new ArrayList<>(); |
|||
for (DynamicValueKey key : dynamicValues.keySet()) { |
|||
switch (key.getSourceType()) { |
|||
case CURRENT_TENANT: |
|||
futures.add(resolveEntityValue(tenantId, tenantId, key)); |
|||
break; |
|||
case CURRENT_CUSTOMER: |
|||
if (customerId != null && !customerId.isNullUid()) { |
|||
futures.add(resolveEntityValue(tenantId, customerId, key)); |
|||
} |
|||
break; |
|||
case CURRENT_USER: |
|||
if (userId != null && !userId.isNullUid()) { |
|||
futures.add(resolveEntityValue(tenantId, userId, key)); |
|||
} |
|||
break; |
|||
} |
|||
} |
|||
try { |
|||
Map<EntityId, Map<String, DynamicValueKeySub>> tmpSubMap = new HashMap<>(); |
|||
for (DynamicValueKeySub sub : Futures.successfulAsList(futures).get()) { |
|||
tmpSubMap.computeIfAbsent(sub.getEntityId(), tmp -> new HashMap<>()).put(sub.getKey().getSourceAttribute(), sub); |
|||
} |
|||
for (EntityId entityId : tmpSubMap.keySet()) { |
|||
Map<String, Long> keyStates = new HashMap<>(); |
|||
Map<String, DynamicValueKeySub> dynamicValueKeySubMap = tmpSubMap.get(entityId); |
|||
dynamicValueKeySubMap.forEach((k, v) -> keyStates.put(k, v.getLastUpdateTs())); |
|||
int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet(); |
|||
TbAttributeSubscription sub = TbAttributeSubscription.builder() |
|||
.serviceId(serviceId) |
|||
.sessionId(sessionRef.getSessionId()) |
|||
.subscriptionId(subIdx) |
|||
.tenantId(sessionRef.getSecurityCtx().getTenantId()) |
|||
.entityId(entityId) |
|||
.updateConsumer((s, subscriptionUpdate) -> dynamicValueSubUpdate(s, subscriptionUpdate, dynamicValueKeySubMap)) |
|||
.allKeys(false) |
|||
.keyStates(keyStates) |
|||
.scope(TbAttributeSubscriptionScope.SERVER_SCOPE) |
|||
.build(); |
|||
subToDynamicValueKeySet.add(subIdx); |
|||
localSubscriptionService.addSubscription(sub); |
|||
} |
|||
} catch (InterruptedException | ExecutionException e) { |
|||
log.info("[{}][{}][{}] Failed to resolve dynamic values: {}", tenantId, customerId, userId, dynamicValues.keySet()); |
|||
} |
|||
|
|||
} |
|||
|
|||
private void dynamicValueSubUpdate(String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, |
|||
Map<String, DynamicValueKeySub> dynamicValueKeySubMap) { |
|||
Map<String, TsValue> latestUpdate = new HashMap<>(); |
|||
subscriptionUpdate.getData().forEach((k, v) -> { |
|||
Object[] data = (Object[]) v.get(0); |
|||
latestUpdate.put(k, new TsValue((Long) data[0], (String) data[1])); |
|||
}); |
|||
|
|||
boolean invalidateFilter = false; |
|||
for (Map.Entry<String, TsValue> entry : latestUpdate.entrySet()) { |
|||
String k = entry.getKey(); |
|||
TsValue tsValue = entry.getValue(); |
|||
DynamicValueKeySub sub = dynamicValueKeySubMap.get(k); |
|||
if (sub.updateValue(tsValue)) { |
|||
invalidateFilter = true; |
|||
updateDynamicValuesByKey(sub, tsValue); |
|||
} |
|||
} |
|||
|
|||
if (invalidateFilter) { |
|||
update(); |
|||
} |
|||
} |
|||
|
|||
public void fetchData() { |
|||
this.data = findEntityData(); |
|||
} |
|||
|
|||
protected PageData<EntityData> findEntityData() { |
|||
PageData<EntityData> result = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); |
|||
if (log.isTraceEnabled()) { |
|||
result.getData().forEach(ed -> { |
|||
log.trace("[{}][{}] EntityData: {}", getSessionId(), getCmdId(), ed); |
|||
}); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
protected synchronized void update() { |
|||
long start = System.currentTimeMillis(); |
|||
PageData<EntityData> newData = findEntityData(); |
|||
long end = System.currentTimeMillis(); |
|||
stats.getRegularQueryInvocationCnt().incrementAndGet(); |
|||
stats.getRegularQueryTimeSpent().addAndGet(end - start); |
|||
Map<EntityId, EntityData> oldDataMap; |
|||
if (data != null && !data.getData().isEmpty()) { |
|||
oldDataMap = data.getData().stream().collect(Collectors.toMap(EntityData::getEntityId, Function.identity(), (a, b) -> a)); |
|||
} else { |
|||
oldDataMap = Collections.emptyMap(); |
|||
} |
|||
Map<EntityId, EntityData> newDataMap = newData.getData().stream().collect(Collectors.toMap(EntityData::getEntityId, Function.identity(), (a,b)-> a)); |
|||
if (oldDataMap.size() == newDataMap.size() && oldDataMap.keySet().equals(newDataMap.keySet())) { |
|||
log.trace("[{}][{}] No updates to entity data found", sessionRef.getSessionId(), cmdId); |
|||
} else { |
|||
this.data = newData; |
|||
doUpdate(newDataMap); |
|||
} |
|||
} |
|||
|
|||
protected abstract void doUpdate(Map<EntityId, EntityData> newDataMap); |
|||
|
|||
protected abstract EntityDataQuery buildEntityDataQuery(); |
|||
|
|||
public List<EntityData> getEntitiesData() { |
|||
return data.getData(); |
|||
} |
|||
|
|||
@Data |
|||
private static class DynamicValueKeySub { |
|||
private final DynamicValueKey key; |
|||
private final EntityId entityId; |
|||
private long lastUpdateTs; |
|||
private String lastUpdateValue; |
|||
|
|||
boolean updateValue(TsValue value) { |
|||
if (value.getTs() > lastUpdateTs && (lastUpdateValue == null || !lastUpdateValue.equals(value.getValue()))) { |
|||
this.lastUpdateTs = value.getTs(); |
|||
this.lastUpdateValue = value.getValue(); |
|||
return true; |
|||
} else { |
|||
return false; |
|||
} |
|||
} |
|||
} |
|||
|
|||
private ListenableFuture<DynamicValueKeySub> resolveEntityValue(TenantId tenantId, EntityId entityId, DynamicValueKey key) { |
|||
ListenableFuture<Optional<AttributeKvEntry>> entry = attributesService.find(tenantId, entityId, |
|||
TbAttributeSubscriptionScope.SERVER_SCOPE.name(), key.getSourceAttribute()); |
|||
return Futures.transform(entry, attributeOpt -> { |
|||
DynamicValueKeySub sub = new DynamicValueKeySub(key, entityId); |
|||
if (attributeOpt.isPresent()) { |
|||
AttributeKvEntry attribute = attributeOpt.get(); |
|||
sub.setLastUpdateTs(attribute.getLastUpdateTs()); |
|||
sub.setLastUpdateValue(attribute.getValueAsString()); |
|||
updateDynamicValuesByKey(sub, new TsValue(attribute.getLastUpdateTs(), attribute.getValueAsString())); |
|||
} |
|||
return sub; |
|||
}, MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
private void updateDynamicValuesByKey(DynamicValueKeySub sub, TsValue tsValue) { |
|||
DynamicValueKey dvk = sub.getKey(); |
|||
switch (dvk.getPredicateType()) { |
|||
case STRING: |
|||
dynamicValues.get(dvk).forEach(dynamicValue -> dynamicValue.setResolvedValue(tsValue.getValue())); |
|||
break; |
|||
case NUMERIC: |
|||
try { |
|||
Double dValue = Double.parseDouble(tsValue.getValue()); |
|||
dynamicValues.get(dvk).forEach(dynamicValue -> dynamicValue.setResolvedValue(dValue)); |
|||
} catch (NumberFormatException e) { |
|||
dynamicValues.get(dvk).forEach(dynamicValue -> dynamicValue.setResolvedValue(null)); |
|||
} |
|||
break; |
|||
case BOOLEAN: |
|||
Boolean bValue = Boolean.parseBoolean(tsValue.getValue()); |
|||
dynamicValues.get(dvk).forEach(dynamicValue -> dynamicValue.setResolvedValue(bValue)); |
|||
break; |
|||
} |
|||
} |
|||
|
|||
private void registerDynamicValues(KeyFilterPredicate predicate) { |
|||
switch (predicate.getType()) { |
|||
case STRING: |
|||
case NUMERIC: |
|||
case BOOLEAN: |
|||
Optional<DynamicValue> value = getDynamicValueFromSimplePredicate((SimpleKeyFilterPredicate) predicate); |
|||
if (value.isPresent()) { |
|||
DynamicValue dynamicValue = value.get(); |
|||
DynamicValueKey key = new DynamicValueKey( |
|||
predicate.getType(), |
|||
dynamicValue.getSourceType(), |
|||
dynamicValue.getSourceAttribute()); |
|||
dynamicValues.computeIfAbsent(key, tmp -> new ArrayList<>()).add(dynamicValue); |
|||
} |
|||
break; |
|||
case COMPLEX: |
|||
((ComplexFilterPredicate) predicate).getPredicates().forEach(this::registerDynamicValues); |
|||
} |
|||
} |
|||
|
|||
private Optional<DynamicValue<T>> getDynamicValueFromSimplePredicate(SimpleKeyFilterPredicate<T> predicate) { |
|||
if (predicate.getValue().getUserValue() == null) { |
|||
return Optional.ofNullable(predicate.getValue().getDynamicValue()); |
|||
} else { |
|||
return Optional.empty(); |
|||
} |
|||
} |
|||
|
|||
public String getSessionId() { |
|||
return sessionRef.getSessionId(); |
|||
} |
|||
|
|||
public TenantId getTenantId() { |
|||
return sessionRef.getSecurityCtx().getTenantId(); |
|||
} |
|||
|
|||
public CustomerId getCustomerId() { |
|||
return sessionRef.getSecurityCtx().getCustomerId(); |
|||
} |
|||
|
|||
public UserId getUserId() { |
|||
return sessionRef.getSecurityCtx().getId(); |
|||
} |
|||
|
|||
public void clearEntitySubscriptions() { |
|||
if (subToEntityIdMap != null) { |
|||
for (Integer subId : subToEntityIdMap.keySet()) { |
|||
localSubscriptionService.cancelSubscription(sessionRef.getSessionId(), subId); |
|||
} |
|||
subToEntityIdMap.clear(); |
|||
} |
|||
} |
|||
|
|||
public void clearDynamicValueSubscriptions() { |
|||
if (subToDynamicValueKeySet != null) { |
|||
for (Integer subId : subToDynamicValueKeySet) { |
|||
localSubscriptionService.cancelSubscription(sessionRef.getSessionId(), subId); |
|||
} |
|||
subToDynamicValueKeySet.clear(); |
|||
} |
|||
} |
|||
|
|||
public void setRefreshTask(ScheduledFuture<?> task) { |
|||
this.refreshTask = task; |
|||
} |
|||
|
|||
public void cancelTasks() { |
|||
if (this.refreshTask != null) { |
|||
log.trace("[{}][{}] Canceling old refresh task", sessionRef.getSessionId(), cmdId); |
|||
this.refreshTask.cancel(true); |
|||
} |
|||
} |
|||
|
|||
public void createSubscriptions(List<EntityKey> keys, boolean resultToLatestValues) { |
|||
Map<EntityKeyType, List<EntityKey>> keysByType = getEntityKeyByTypeMap(keys); |
|||
for (EntityData entityData : data.getData()) { |
|||
List<TbSubscription> entitySubscriptions = addSubscriptions(entityData, keysByType, resultToLatestValues); |
|||
entitySubscriptions.forEach(localSubscriptionService::addSubscription); |
|||
} |
|||
} |
|||
|
|||
protected Map<EntityKeyType, List<EntityKey>> getEntityKeyByTypeMap(List<EntityKey> keys) { |
|||
Map<EntityKeyType, List<EntityKey>> keysByType = new HashMap<>(); |
|||
keys.forEach(key -> keysByType.computeIfAbsent(key.getType(), k -> new ArrayList<>()).add(key)); |
|||
return keysByType; |
|||
} |
|||
|
|||
protected List<TbSubscription> addSubscriptions(EntityData entityData, Map<EntityKeyType, List<EntityKey>> keysByType, boolean resultToLatestValues) { |
|||
List<TbSubscription> subscriptionList = new ArrayList<>(); |
|||
keysByType.forEach((keysType, keysList) -> { |
|||
int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet(); |
|||
subToEntityIdMap.put(subIdx, entityData.getEntityId()); |
|||
switch (keysType) { |
|||
case TIME_SERIES: |
|||
subscriptionList.add(createTsSub(entityData, subIdx, keysList, resultToLatestValues)); |
|||
break; |
|||
case CLIENT_ATTRIBUTE: |
|||
subscriptionList.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.CLIENT_SCOPE, keysList)); |
|||
break; |
|||
case SHARED_ATTRIBUTE: |
|||
subscriptionList.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.SHARED_SCOPE, keysList)); |
|||
break; |
|||
case SERVER_ATTRIBUTE: |
|||
subscriptionList.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.SERVER_SCOPE, keysList)); |
|||
break; |
|||
case ATTRIBUTE: |
|||
subscriptionList.add(createAttrSub(entityData, subIdx, keysType, TbAttributeSubscriptionScope.ANY_SCOPE, keysList)); |
|||
break; |
|||
} |
|||
}); |
|||
return subscriptionList; |
|||
} |
|||
|
|||
private TbSubscription createAttrSub(EntityData entityData, int subIdx, EntityKeyType keysType, TbAttributeSubscriptionScope scope, List<EntityKey> subKeys) { |
|||
Map<String, Long> keyStates = buildKeyStats(entityData, keysType, subKeys); |
|||
log.trace("[{}][{}][{}] Creating attributes subscription for [{}] with keys: {}", serviceId, cmdId, subIdx, entityData.getEntityId(), keyStates); |
|||
return TbAttributeSubscription.builder() |
|||
.serviceId(serviceId) |
|||
.sessionId(sessionRef.getSessionId()) |
|||
.subscriptionId(subIdx) |
|||
.tenantId(sessionRef.getSecurityCtx().getTenantId()) |
|||
.entityId(entityData.getEntityId()) |
|||
.updateConsumer((s, subscriptionUpdate) -> sendWsMsg(s, subscriptionUpdate, keysType)) |
|||
.allKeys(false) |
|||
.keyStates(keyStates) |
|||
.scope(scope) |
|||
.build(); |
|||
} |
|||
|
|||
private TbSubscription createTsSub(EntityData entityData, int subIdx, List<EntityKey> subKeys, boolean resultToLatestValues) { |
|||
Map<String, Long> keyStates = buildKeyStats(entityData, EntityKeyType.TIME_SERIES, subKeys); |
|||
if (entityData.getTimeseries() != null) { |
|||
entityData.getTimeseries().forEach((k, v) -> { |
|||
long ts = Arrays.stream(v).map(TsValue::getTs).max(Long::compareTo).orElse(0L); |
|||
log.trace("[{}][{}] Updating key: {} with ts: {}", serviceId, cmdId, k, ts); |
|||
keyStates.put(k, ts); |
|||
}); |
|||
} |
|||
log.trace("[{}][{}][{}] Creating time-series subscription for [{}] with keys: {}", serviceId, cmdId, subIdx, entityData.getEntityId(), keyStates); |
|||
return TbTimeseriesSubscription.builder() |
|||
.serviceId(serviceId) |
|||
.sessionId(sessionRef.getSessionId()) |
|||
.subscriptionId(subIdx) |
|||
.tenantId(sessionRef.getSecurityCtx().getTenantId()) |
|||
.entityId(entityData.getEntityId()) |
|||
.updateConsumer((sessionId, subscriptionUpdate) -> sendWsMsg(sessionId, subscriptionUpdate, EntityKeyType.TIME_SERIES, resultToLatestValues)) |
|||
.allKeys(false) |
|||
.keyStates(keyStates) |
|||
.build(); |
|||
} |
|||
|
|||
private void sendWsMsg(String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { |
|||
sendWsMsg(sessionId, subscriptionUpdate, keyType, true); |
|||
} |
|||
|
|||
private Map<String, Long> buildKeyStats(EntityData entityData, EntityKeyType keysType, List<EntityKey> subKeys) { |
|||
Map<String, Long> keyStates = new HashMap<>(); |
|||
subKeys.forEach(key -> keyStates.put(key.getKey(), 0L)); |
|||
if (entityData.getLatest() != null) { |
|||
Map<String, TsValue> currentValues = entityData.getLatest().get(keysType); |
|||
if (currentValues != null) { |
|||
currentValues.forEach((k, v) -> { |
|||
log.trace("[{}][{}] Updating key: {} with ts: {}", serviceId, cmdId, k, v.getTs()); |
|||
keyStates.put(k, v.getTs()); |
|||
}); |
|||
} |
|||
} |
|||
return keyStates; |
|||
} |
|||
|
|||
abstract void sendWsMsg(String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType, boolean resultToLatestValues); |
|||
|
|||
@Data |
|||
private static class DynamicValueKey { |
|||
@Getter |
|||
private final FilterPredicateType predicateType; |
|||
@Getter |
|||
private final DynamicValueSourceType sourceType; |
|||
@Getter |
|||
private final String sourceAttribute; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,309 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.subscription; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.Setter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.common.data.alarm.Alarm; |
|||
import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; |
|||
import org.thingsboard.server.common.data.id.AlarmId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AlarmData; |
|||
import org.thingsboard.server.common.data.query.AlarmDataPageLink; |
|||
import org.thingsboard.server.common.data.query.AlarmDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataPageLink; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityDataSortOrder; |
|||
import org.thingsboard.server.common.data.query.EntityKey; |
|||
import org.thingsboard.server.common.data.query.EntityKeyType; |
|||
import org.thingsboard.server.common.data.query.TsValue; |
|||
import org.thingsboard.server.dao.alarm.AlarmService; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.entity.EntityService; |
|||
import org.thingsboard.server.dao.model.ModelConstants; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.AlarmDataUpdate; |
|||
import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; |
|||
import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.LinkedHashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.function.Function; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Slf4j |
|||
public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx<AlarmDataQuery> { |
|||
|
|||
private final AlarmService alarmService; |
|||
@Getter |
|||
@Setter |
|||
private final LinkedHashMap<EntityId, EntityData> entitiesMap; |
|||
@Getter |
|||
@Setter |
|||
private final HashMap<AlarmId, AlarmData> alarmsMap; |
|||
|
|||
private final int maxEntitiesPerAlarmSubscription; |
|||
|
|||
@Getter |
|||
@Setter |
|||
private PageData<AlarmData> alarms; |
|||
@Getter |
|||
@Setter |
|||
private boolean tooManyEntities; |
|||
|
|||
public TbAlarmDataSubCtx(String serviceId, TelemetryWebSocketService wsService, |
|||
EntityService entityService, TbLocalSubscriptionService localSubscriptionService, |
|||
AttributesService attributesService, SubscriptionServiceStatistics stats, AlarmService alarmService, |
|||
TelemetryWebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription) { |
|||
super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId); |
|||
this.maxEntitiesPerAlarmSubscription = maxEntitiesPerAlarmSubscription; |
|||
this.alarmService = alarmService; |
|||
this.entitiesMap = new LinkedHashMap<>(); |
|||
this.alarmsMap = new HashMap<>(); |
|||
} |
|||
|
|||
public void fetchAlarms() { |
|||
AlarmDataUpdate update; |
|||
if (!entitiesMap.isEmpty()) { |
|||
long start = System.currentTimeMillis(); |
|||
PageData<AlarmData> alarms = alarmService.findAlarmDataByQueryForEntities(getTenantId(), getCustomerId(), |
|||
query, getOrderedEntityIds()); |
|||
long end = System.currentTimeMillis(); |
|||
stats.getAlarmQueryInvocationCnt().incrementAndGet(); |
|||
stats.getAlarmQueryTimeSpent().addAndGet(end - start); |
|||
alarms = setAndMergeAlarmsData(alarms); |
|||
update = new AlarmDataUpdate(cmdId, alarms, null, maxEntitiesPerAlarmSubscription, data.getTotalElements()); |
|||
} else { |
|||
update = new AlarmDataUpdate(cmdId, new PageData<>(), null, maxEntitiesPerAlarmSubscription, data.getTotalElements()); |
|||
} |
|||
wsService.sendWsMsg(getSessionId(), update); |
|||
} |
|||
|
|||
public void fetchData() { |
|||
super.fetchData(); |
|||
entitiesMap.clear(); |
|||
tooManyEntities = data.hasNext(); |
|||
for (EntityData entityData : data.getData()) { |
|||
entitiesMap.put(entityData.getEntityId(), entityData); |
|||
} |
|||
} |
|||
|
|||
public Collection<EntityId> getOrderedEntityIds() { |
|||
return entitiesMap.keySet(); |
|||
} |
|||
|
|||
public PageData<AlarmData> setAndMergeAlarmsData(PageData<AlarmData> alarms) { |
|||
this.alarms = alarms; |
|||
for (AlarmData alarmData : alarms.getData()) { |
|||
EntityId entityId = alarmData.getEntityId(); |
|||
if (entityId != null) { |
|||
EntityData entityData = entitiesMap.get(entityId); |
|||
if (entityData != null) { |
|||
alarmData.getLatest().putAll(entityData.getLatest()); |
|||
} |
|||
} |
|||
} |
|||
alarmsMap.clear(); |
|||
alarmsMap.putAll(alarms.getData().stream().collect(Collectors.toMap(AlarmData::getId, Function.identity(), (a, b) -> a))); |
|||
return this.alarms; |
|||
} |
|||
|
|||
@Override |
|||
public void createSubscriptions(List<EntityKey> keys, boolean resultToLatestValues) { |
|||
super.createSubscriptions(keys, resultToLatestValues); |
|||
createAlarmSubscriptions(); |
|||
} |
|||
|
|||
public void createAlarmSubscriptions() { |
|||
AlarmDataPageLink pageLink = query.getPageLink(); |
|||
long startTs = System.currentTimeMillis() - pageLink.getTimeWindow(); |
|||
for (EntityData entityData : entitiesMap.values()) { |
|||
createAlarmSubscriptionForEntity(pageLink, startTs, entityData); |
|||
} |
|||
} |
|||
|
|||
private void createAlarmSubscriptionForEntity(AlarmDataPageLink pageLink, long startTs, EntityData entityData) { |
|||
int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet(); |
|||
subToEntityIdMap.put(subIdx, entityData.getEntityId()); |
|||
log.trace("[{}][{}][{}] Creating alarms subscription for [{}] with query: {}", serviceId, cmdId, subIdx, entityData.getEntityId(), pageLink); |
|||
TbAlarmsSubscription subscription = TbAlarmsSubscription.builder() |
|||
.serviceId(serviceId) |
|||
.sessionId(sessionRef.getSessionId()) |
|||
.subscriptionId(subIdx) |
|||
.tenantId(sessionRef.getSecurityCtx().getTenantId()) |
|||
.entityId(entityData.getEntityId()) |
|||
.updateConsumer(this::sendWsMsg) |
|||
.ts(startTs) |
|||
.build(); |
|||
localSubscriptionService.addSubscription(subscription); |
|||
} |
|||
|
|||
@Override |
|||
void sendWsMsg(String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType, boolean resultToLatestValues) { |
|||
EntityId entityId = subToEntityIdMap.get(subscriptionUpdate.getSubscriptionId()); |
|||
if (entityId != null) { |
|||
Map<String, TsValue> latestUpdate = new HashMap<>(); |
|||
subscriptionUpdate.getData().forEach((k, v) -> { |
|||
Object[] data = (Object[]) v.get(0); |
|||
latestUpdate.put(k, new TsValue((Long) data[0], (String) data[1])); |
|||
}); |
|||
EntityData entityData = entitiesMap.get(entityId); |
|||
entityData.getLatest().computeIfAbsent(keyType, tmp -> new HashMap<>()).putAll(latestUpdate); |
|||
log.trace("[{}][{}][{}][{}] Received subscription update: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), keyType, subscriptionUpdate); |
|||
List<AlarmData> update = alarmsMap.values().stream().filter(alarm -> entityId.equals(alarm.getEntityId())).map(alarm -> { |
|||
alarm.getLatest().computeIfAbsent(keyType, tmp -> new HashMap<>()).putAll(latestUpdate); |
|||
return alarm; |
|||
}).collect(Collectors.toList()); |
|||
wsService.sendWsMsg(sessionId, new AlarmDataUpdate(cmdId, null, update, maxEntitiesPerAlarmSubscription, data.getTotalElements())); |
|||
} else { |
|||
log.trace("[{}][{}][{}][{}] Received stale subscription update: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), keyType, subscriptionUpdate); |
|||
} |
|||
} |
|||
|
|||
private void sendWsMsg(String sessionId, AlarmSubscriptionUpdate subscriptionUpdate) { |
|||
Alarm alarm = subscriptionUpdate.getAlarm(); |
|||
AlarmId alarmId = alarm.getId(); |
|||
if (subscriptionUpdate.isAlarmDeleted()) { |
|||
Alarm deleted = alarmsMap.remove(alarmId); |
|||
if (deleted != null) { |
|||
fetchAlarms(); |
|||
} |
|||
} else { |
|||
AlarmData current = alarmsMap.get(alarmId); |
|||
boolean onCurrentPage = current != null; |
|||
boolean matchesFilter = filter(alarm); |
|||
if (onCurrentPage) { |
|||
if (matchesFilter) { |
|||
AlarmData updated = new AlarmData(alarm, current.getOriginatorName(), current.getEntityId()); |
|||
updated.getLatest().putAll(current.getLatest()); |
|||
alarmsMap.put(alarmId, updated); |
|||
wsService.sendWsMsg(sessionId, new AlarmDataUpdate(cmdId, null, Collections.singletonList(updated), maxEntitiesPerAlarmSubscription, data.getTotalElements())); |
|||
} else { |
|||
fetchAlarms(); |
|||
} |
|||
} else if (matchesFilter && query.getPageLink().getPage() == 0) { |
|||
fetchAlarms(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void cleanupOldAlarms() { |
|||
long expTime = System.currentTimeMillis() - query.getPageLink().getTimeWindow(); |
|||
boolean shouldRefresh = false; |
|||
for (AlarmData alarmData : alarms.getData()) { |
|||
if (alarmData.getCreatedTime() < expTime) { |
|||
shouldRefresh = true; |
|||
break; |
|||
} |
|||
} |
|||
if (shouldRefresh) { |
|||
fetchAlarms(); |
|||
} |
|||
} |
|||
|
|||
private boolean filter(Alarm alarm) { |
|||
AlarmDataPageLink filter = query.getPageLink(); |
|||
long startTs = System.currentTimeMillis() - filter.getTimeWindow(); |
|||
if (alarm.getCreatedTime() < startTs) { |
|||
//Skip update that does not match time window.
|
|||
return false; |
|||
} |
|||
if (filter.getTypeList() != null && !filter.getTypeList().isEmpty() && !filter.getTypeList().contains(alarm.getType())) { |
|||
return false; |
|||
} |
|||
if (filter.getSeverityList() != null && !filter.getSeverityList().isEmpty()) { |
|||
if (!filter.getSeverityList().contains(alarm.getSeverity())) { |
|||
return false; |
|||
} |
|||
} |
|||
if (filter.getStatusList() != null && !filter.getStatusList().isEmpty()) { |
|||
boolean matches = false; |
|||
for (AlarmSearchStatus status : filter.getStatusList()) { |
|||
if (status.getStatuses().contains(alarm.getStatus())) { |
|||
matches = true; |
|||
break; |
|||
} |
|||
} |
|||
if (!matches) { |
|||
return false; |
|||
} |
|||
} |
|||
return true; |
|||
} |
|||
|
|||
@Override |
|||
protected synchronized void doUpdate(Map<EntityId, EntityData> newDataMap) { |
|||
entitiesMap.clear(); |
|||
tooManyEntities = data.hasNext(); |
|||
for (EntityData entityData : data.getData()) { |
|||
entitiesMap.put(entityData.getEntityId(), entityData); |
|||
} |
|||
fetchAlarms(); |
|||
List<Integer> subIdsToCancel = new ArrayList<>(); |
|||
List<TbSubscription> subsToAdd = new ArrayList<>(); |
|||
Set<EntityId> currentSubs = new HashSet<>(); |
|||
subToEntityIdMap.forEach((subId, entityId) -> { |
|||
if (!newDataMap.containsKey(entityId)) { |
|||
subIdsToCancel.add(subId); |
|||
} else { |
|||
currentSubs.add(entityId); |
|||
} |
|||
}); |
|||
log.trace("[{}][{}] Subscriptions that are invalid: {}", sessionRef.getSessionId(), cmdId, subIdsToCancel); |
|||
subIdsToCancel.forEach(subToEntityIdMap::remove); |
|||
List<EntityData> newSubsList = newDataMap.entrySet().stream().filter(entry -> !currentSubs.contains(entry.getKey())).map(Map.Entry::getValue).collect(Collectors.toList()); |
|||
if (!newSubsList.isEmpty()) { |
|||
List<EntityKey> keys = query.getLatestValues(); |
|||
if (keys != null && !keys.isEmpty()) { |
|||
Map<EntityKeyType, List<EntityKey>> keysByType = getEntityKeyByTypeMap(keys); |
|||
newSubsList.forEach( |
|||
entity -> { |
|||
log.trace("[{}][{}] Found new subscription for entity: {}", sessionRef.getSessionId(), cmdId, entity.getEntityId()); |
|||
subsToAdd.addAll(addSubscriptions(entity, keysByType, true)); |
|||
} |
|||
); |
|||
} |
|||
long startTs = System.currentTimeMillis() - query.getPageLink().getTimeWindow(); |
|||
newSubsList.forEach(entity -> createAlarmSubscriptionForEntity(query.getPageLink(), startTs, entity)); |
|||
} |
|||
subIdsToCancel.forEach(subId -> localSubscriptionService.cancelSubscription(getSessionId(), subId)); |
|||
subsToAdd.forEach(localSubscriptionService::addSubscription); |
|||
} |
|||
|
|||
@Override |
|||
protected EntityDataQuery buildEntityDataQuery() { |
|||
EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); |
|||
EntityDataSortOrder entitiesSortOrder; |
|||
if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { |
|||
entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); |
|||
} else { |
|||
entitiesSortOrder = sortOrder; |
|||
} |
|||
EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, entitiesSortOrder); |
|||
return new EntityDataQuery(query.getEntityFilter(), edpl, query.getEntityFields(), query.getLatestValues(), query.getKeyFilters()); |
|||
} |
|||
} |
|||
@ -0,0 +1,50 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.subscription; |
|||
|
|||
import lombok.Builder; |
|||
import lombok.Getter; |
|||
import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; |
|||
import org.thingsboard.server.common.data.alarm.AlarmSeverity; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; |
|||
|
|||
import java.util.List; |
|||
import java.util.function.BiConsumer; |
|||
|
|||
public class TbAlarmsSubscription extends TbSubscription<AlarmSubscriptionUpdate> { |
|||
|
|||
@Getter |
|||
private final long ts; |
|||
|
|||
@Builder |
|||
public TbAlarmsSubscription(String serviceId, String sessionId, int subscriptionId, TenantId tenantId, EntityId entityId, |
|||
BiConsumer<String, AlarmSubscriptionUpdate> updateConsumer, long ts) { |
|||
super(serviceId, sessionId, subscriptionId, tenantId, entityId, TbSubscriptionType.ALARMS, updateConsumer); |
|||
this.ts = ts; |
|||
} |
|||
|
|||
@Override |
|||
public boolean equals(Object o) { |
|||
return super.equals(o); |
|||
} |
|||
|
|||
@Override |
|||
public int hashCode() { |
|||
return super.hashCode(); |
|||
} |
|||
} |
|||
@ -0,0 +1,220 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.subscription; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.Setter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityKey; |
|||
import org.thingsboard.server.common.data.query.EntityKeyType; |
|||
import org.thingsboard.server.common.data.query.TsValue; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.entity.EntityService; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService; |
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.LatestValueCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.TimeSeriesCmd; |
|||
import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.Comparator; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Optional; |
|||
import java.util.Set; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Slf4j |
|||
public class TbEntityDataSubCtx extends TbAbstractDataSubCtx<EntityDataQuery> { |
|||
|
|||
@Getter |
|||
@Setter |
|||
private TimeSeriesCmd tsCmd; |
|||
@Getter |
|||
@Setter |
|||
private boolean initialDataSent; |
|||
private TimeSeriesCmd curTsCmd; |
|||
private LatestValueCmd latestValueCmd; |
|||
@Getter |
|||
private final int maxEntitiesPerDataSubscription; |
|||
|
|||
public TbEntityDataSubCtx(String serviceId, TelemetryWebSocketService wsService, EntityService entityService, |
|||
TbLocalSubscriptionService localSubscriptionService, AttributesService attributesService, |
|||
SubscriptionServiceStatistics stats, TelemetryWebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerDataSubscription) { |
|||
super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId); |
|||
this.maxEntitiesPerDataSubscription = maxEntitiesPerDataSubscription; |
|||
} |
|||
|
|||
@Override |
|||
protected void sendWsMsg(String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType, boolean resultToLatestValues) { |
|||
EntityId entityId = subToEntityIdMap.get(subscriptionUpdate.getSubscriptionId()); |
|||
if (entityId != null) { |
|||
log.trace("[{}][{}][{}][{}] Received subscription update: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), keyType, subscriptionUpdate); |
|||
if (resultToLatestValues) { |
|||
sendLatestWsMsg(entityId, sessionId, subscriptionUpdate, keyType); |
|||
} else { |
|||
sendTsWsMsg(entityId, sessionId, subscriptionUpdate, keyType); |
|||
} |
|||
} else { |
|||
log.trace("[{}][{}][{}][{}] Received stale subscription update: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), keyType, subscriptionUpdate); |
|||
} |
|||
} |
|||
|
|||
private void sendLatestWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { |
|||
Map<String, TsValue> latestUpdate = new HashMap<>(); |
|||
subscriptionUpdate.getData().forEach((k, v) -> { |
|||
Object[] data = (Object[]) v.get(0); |
|||
latestUpdate.put(k, new TsValue((Long) data[0], (String) data[1])); |
|||
}); |
|||
EntityData entityData = getDataForEntity(entityId); |
|||
if (entityData != null && entityData.getLatest() != null) { |
|||
Map<String, TsValue> latestCtxValues = entityData.getLatest().get(keyType); |
|||
log.trace("[{}][{}][{}] Going to compare update with {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), latestCtxValues); |
|||
if (latestCtxValues != null) { |
|||
latestCtxValues.forEach((k, v) -> { |
|||
TsValue update = latestUpdate.get(k); |
|||
if (update != null) { |
|||
//Ignore notifications about deleted keys
|
|||
if (!(update.getTs() == 0 && (update.getValue() == null || update.getValue().isEmpty()))) { |
|||
if (update.getTs() < v.getTs()) { |
|||
log.trace("[{}][{}][{}] Removed stale update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); |
|||
latestUpdate.remove(k); |
|||
} else if ((update.getTs() == v.getTs() && update.getValue().equals(v.getValue()))) { |
|||
log.trace("[{}][{}][{}] Removed duplicate update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); |
|||
latestUpdate.remove(k); |
|||
} |
|||
} else { |
|||
log.trace("[{}][{}][{}] Received deleted notification for: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k); |
|||
} |
|||
} |
|||
}); |
|||
//Setting new values
|
|||
latestUpdate.forEach(latestCtxValues::put); |
|||
} |
|||
} |
|||
if (!latestUpdate.isEmpty()) { |
|||
Map<EntityKeyType, Map<String, TsValue>> latestMap = Collections.singletonMap(keyType, latestUpdate); |
|||
entityData = new EntityData(entityId, latestMap, null); |
|||
wsService.sendWsMsg(sessionId, new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData), maxEntitiesPerDataSubscription)); |
|||
} |
|||
} |
|||
|
|||
private void sendTsWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { |
|||
Map<String, List<TsValue>> tsUpdate = new HashMap<>(); |
|||
subscriptionUpdate.getData().forEach((k, v) -> { |
|||
Object[] data = (Object[]) v.get(0); |
|||
tsUpdate.computeIfAbsent(k, key -> new ArrayList<>()).add(new TsValue((Long) data[0], (String) data[1])); |
|||
}); |
|||
EntityData entityData = getDataForEntity(entityId); |
|||
if (entityData != null && entityData.getLatest() != null) { |
|||
Map<String, TsValue> latestCtxValues = entityData.getLatest().get(keyType); |
|||
log.trace("[{}][{}][{}] Going to compare update with {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), latestCtxValues); |
|||
if (latestCtxValues != null) { |
|||
latestCtxValues.forEach((k, v) -> { |
|||
List<TsValue> updateList = tsUpdate.get(k); |
|||
if (updateList != null) { |
|||
for (TsValue update : new ArrayList<>(updateList)) { |
|||
if (update.getTs() < v.getTs()) { |
|||
log.trace("[{}][{}][{}] Removed stale update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); |
|||
updateList.remove(update); |
|||
} else if ((update.getTs() == v.getTs() && update.getValue().equals(v.getValue()))) { |
|||
log.trace("[{}][{}][{}] Removed duplicate update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); |
|||
updateList.remove(update); |
|||
} |
|||
if (updateList.isEmpty()) { |
|||
tsUpdate.remove(k); |
|||
} |
|||
} |
|||
} |
|||
}); |
|||
//Setting new values
|
|||
tsUpdate.forEach((k, v) -> { |
|||
Optional<TsValue> maxValue = v.stream().max(Comparator.comparingLong(TsValue::getTs)); |
|||
maxValue.ifPresent(max -> latestCtxValues.put(k, max)); |
|||
}); |
|||
} |
|||
} |
|||
if (!tsUpdate.isEmpty()) { |
|||
Map<String, TsValue[]> tsMap = new HashMap<>(); |
|||
tsUpdate.forEach((key, tsValue) -> tsMap.put(key, tsValue.toArray(new TsValue[tsValue.size()]))); |
|||
entityData = new EntityData(entityId, null, tsMap); |
|||
wsService.sendWsMsg(sessionId, new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData), maxEntitiesPerDataSubscription)); |
|||
} |
|||
} |
|||
|
|||
private EntityData getDataForEntity(EntityId entityId) { |
|||
return data.getData().stream().filter(item -> item.getEntityId().equals(entityId)).findFirst().orElse(null); |
|||
} |
|||
|
|||
public synchronized void doUpdate(Map<EntityId, EntityData> newDataMap) { |
|||
List<Integer> subIdsToCancel = new ArrayList<>(); |
|||
List<TbSubscription> subsToAdd = new ArrayList<>(); |
|||
Set<EntityId> currentSubs = new HashSet<>(); |
|||
subToEntityIdMap.forEach((subId, entityId) -> { |
|||
if (!newDataMap.containsKey(entityId)) { |
|||
subIdsToCancel.add(subId); |
|||
} else { |
|||
currentSubs.add(entityId); |
|||
} |
|||
}); |
|||
log.trace("[{}][{}] Subscriptions that are invalid: {}", sessionRef.getSessionId(), cmdId, subIdsToCancel); |
|||
subIdsToCancel.forEach(subToEntityIdMap::remove); |
|||
List<EntityData> newSubsList = newDataMap.entrySet().stream().filter(entry -> !currentSubs.contains(entry.getKey())).map(Map.Entry::getValue).collect(Collectors.toList()); |
|||
if (!newSubsList.isEmpty()) { |
|||
boolean resultToLatestValues; |
|||
List<EntityKey> keys = null; |
|||
if (curTsCmd != null) { |
|||
resultToLatestValues = false; |
|||
keys = curTsCmd.getKeys().stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).collect(Collectors.toList()); |
|||
} else if (latestValueCmd != null) { |
|||
resultToLatestValues = true; |
|||
keys = latestValueCmd.getKeys(); |
|||
} else { |
|||
resultToLatestValues = true; |
|||
} |
|||
if (keys != null && !keys.isEmpty()) { |
|||
Map<EntityKeyType, List<EntityKey>> keysByType = getEntityKeyByTypeMap(keys); |
|||
newSubsList.forEach( |
|||
entity -> { |
|||
log.trace("[{}][{}] Found new subscription for entity: {}", sessionRef.getSessionId(), cmdId, entity.getEntityId()); |
|||
subsToAdd.addAll(addSubscriptions(entity, keysByType, resultToLatestValues)); |
|||
} |
|||
); |
|||
} |
|||
} |
|||
wsService.sendWsMsg(sessionRef.getSessionId(), new EntityDataUpdate(cmdId, data, null, maxEntitiesPerDataSubscription)); |
|||
subIdsToCancel.forEach(subId -> localSubscriptionService.cancelSubscription(getSessionId(), subId)); |
|||
subsToAdd.forEach(localSubscriptionService::addSubscription); |
|||
} |
|||
|
|||
public void setCurrentCmd(EntityDataCmd cmd) { |
|||
curTsCmd = cmd.getTsCmd(); |
|||
latestValueCmd = cmd.getLatestCmd(); |
|||
} |
|||
|
|||
@Override |
|||
protected EntityDataQuery buildEntityDataQuery() { |
|||
return query; |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.subscription; |
|||
|
|||
import org.thingsboard.server.service.telemetry.TelemetryWebSocketSessionRef; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.AlarmDataCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUnsubscribeCmd; |
|||
import org.thingsboard.server.service.telemetry.cmd.v2.UnsubscribeCmd; |
|||
|
|||
public interface TbEntityDataSubscriptionService { |
|||
|
|||
void handleCmd(TelemetryWebSocketSessionRef sessionId, EntityDataCmd cmd); |
|||
|
|||
void handleCmd(TelemetryWebSocketSessionRef sessionId, AlarmDataCmd cmd); |
|||
|
|||
void cancelSubscription(String sessionId, UnsubscribeCmd subscriptionId); |
|||
|
|||
void cancelAllSessionSubscriptions(String sessionId); |
|||
|
|||
} |
|||
@ -0,0 +1,120 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.telemetry; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.context.ApplicationListener; |
|||
import org.springframework.context.event.EventListener; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BooleanDataEntry; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.LongDataEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos; |
|||
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.service.queue.TbClusterService; |
|||
import org.thingsboard.server.service.subscription.SubscriptionManagerService; |
|||
import org.thingsboard.server.service.subscription.TbSubscriptionUtils; |
|||
|
|||
import javax.annotation.Nullable; |
|||
import javax.annotation.PostConstruct; |
|||
import javax.annotation.PreDestroy; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
import java.util.Set; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.function.Consumer; |
|||
|
|||
/** |
|||
* Created by ashvayka on 27.03.18. |
|||
*/ |
|||
@Slf4j |
|||
public abstract class AbstractSubscriptionService implements ApplicationListener<PartitionChangeEvent> { |
|||
|
|||
protected final Set<TopicPartitionInfo> currentPartitions = ConcurrentHashMap.newKeySet(); |
|||
|
|||
protected final TbClusterService clusterService; |
|||
protected final PartitionService partitionService; |
|||
protected Optional<SubscriptionManagerService> subscriptionManagerService; |
|||
|
|||
protected ExecutorService wsCallBackExecutor; |
|||
|
|||
public AbstractSubscriptionService(TbClusterService clusterService, |
|||
PartitionService partitionService) { |
|||
this.clusterService = clusterService; |
|||
this.partitionService = partitionService; |
|||
} |
|||
|
|||
@Autowired(required = false) |
|||
public void setSubscriptionManagerService(Optional<SubscriptionManagerService> subscriptionManagerService) { |
|||
this.subscriptionManagerService = subscriptionManagerService; |
|||
} |
|||
|
|||
abstract String getExecutorPrefix(); |
|||
|
|||
@PostConstruct |
|||
public void initExecutor() { |
|||
wsCallBackExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(getExecutorPrefix() + "-service-ws-callback")); |
|||
} |
|||
|
|||
@PreDestroy |
|||
public void shutdownExecutor() { |
|||
if (wsCallBackExecutor != null) { |
|||
wsCallBackExecutor.shutdownNow(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
@EventListener(PartitionChangeEvent.class) |
|||
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { |
|||
if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { |
|||
currentPartitions.clear(); |
|||
currentPartitions.addAll(partitionChangeEvent.getPartitions()); |
|||
} |
|||
} |
|||
|
|||
protected void addWsCallback(ListenableFuture<List<Void>> saveFuture, Consumer<Void> callback) { |
|||
Futures.addCallback(saveFuture, new FutureCallback<List<Void>>() { |
|||
@Override |
|||
public void onSuccess(@Nullable List<Void> result) { |
|||
callback.accept(null); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
} |
|||
}, wsCallBackExecutor); |
|||
} |
|||
} |
|||
@ -0,0 +1,28 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.telemetry; |
|||
|
|||
import org.springframework.context.ApplicationListener; |
|||
import org.thingsboard.rule.engine.api.RuleEngineAlarmService; |
|||
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; |
|||
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; |
|||
|
|||
/** |
|||
* Created by ashvayka on 27.03.18. |
|||
*/ |
|||
public interface AlarmSubscriptionService extends RuleEngineAlarmService, ApplicationListener<PartitionChangeEvent> { |
|||
|
|||
} |
|||
@ -0,0 +1,214 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.telemetry; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.checkerframework.checker.nullness.qual.Nullable; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.context.event.EventListener; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.alarm.Alarm; |
|||
import org.thingsboard.server.common.data.alarm.AlarmInfo; |
|||
import org.thingsboard.server.common.data.alarm.AlarmQuery; |
|||
import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; |
|||
import org.thingsboard.server.common.data.alarm.AlarmSeverity; |
|||
import org.thingsboard.server.common.data.alarm.AlarmStatus; |
|||
import org.thingsboard.server.common.data.id.AlarmId; |
|||
import org.thingsboard.server.common.data.id.CustomerId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BooleanDataEntry; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.LongDataEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AlarmData; |
|||
import org.thingsboard.server.common.data.query.AlarmDataPageLink; |
|||
import org.thingsboard.server.common.data.query.AlarmDataQuery; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.alarm.AlarmOperationResult; |
|||
import org.thingsboard.server.dao.alarm.AlarmService; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos; |
|||
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.service.queue.TbClusterService; |
|||
import org.thingsboard.server.service.subscription.SubscriptionManagerService; |
|||
import org.thingsboard.server.service.subscription.TbSubscriptionUtils; |
|||
import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; |
|||
|
|||
import javax.annotation.PostConstruct; |
|||
import javax.annotation.PreDestroy; |
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
import java.util.Set; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.function.Consumer; |
|||
|
|||
/** |
|||
* Created by ashvayka on 27.03.18. |
|||
*/ |
|||
@Service |
|||
@Slf4j |
|||
public class DefaultAlarmSubscriptionService extends AbstractSubscriptionService implements AlarmSubscriptionService { |
|||
|
|||
private final AlarmService alarmService; |
|||
|
|||
public DefaultAlarmSubscriptionService(TbClusterService clusterService, |
|||
PartitionService partitionService, |
|||
AlarmService alarmService) { |
|||
super(clusterService, partitionService); |
|||
this.alarmService = alarmService; |
|||
} |
|||
|
|||
@Autowired(required = false) |
|||
public void setSubscriptionManagerService(Optional<SubscriptionManagerService> subscriptionManagerService) { |
|||
this.subscriptionManagerService = subscriptionManagerService; |
|||
} |
|||
|
|||
@Override |
|||
String getExecutorPrefix() { |
|||
return "alarm"; |
|||
} |
|||
|
|||
@Override |
|||
public Alarm createOrUpdateAlarm(Alarm alarm) { |
|||
AlarmOperationResult result = alarmService.createOrUpdateAlarm(alarm); |
|||
if (result.isSuccessful()) { |
|||
onAlarmUpdated(result); |
|||
} |
|||
return result.getAlarm(); |
|||
} |
|||
|
|||
@Override |
|||
public Boolean deleteAlarm(TenantId tenantId, AlarmId alarmId) { |
|||
AlarmOperationResult result = alarmService.deleteAlarm(tenantId, alarmId); |
|||
onAlarmDeleted(result); |
|||
return result.isSuccessful(); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Boolean> ackAlarm(TenantId tenantId, AlarmId alarmId, long ackTs) { |
|||
ListenableFuture<AlarmOperationResult> result = alarmService.ackAlarm(tenantId, alarmId, ackTs); |
|||
Futures.addCallback(result, new AlarmUpdateCallback(), wsCallBackExecutor); |
|||
return Futures.transform(result, AlarmOperationResult::isSuccessful, wsCallBackExecutor); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Boolean> clearAlarm(TenantId tenantId, AlarmId alarmId, JsonNode details, long clearTs) { |
|||
ListenableFuture<AlarmOperationResult> result = alarmService.clearAlarm(tenantId, alarmId, details, clearTs); |
|||
Futures.addCallback(result, new AlarmUpdateCallback(), wsCallBackExecutor); |
|||
return Futures.transform(result, AlarmOperationResult::isSuccessful, wsCallBackExecutor); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Alarm> findAlarmByIdAsync(TenantId tenantId, AlarmId alarmId) { |
|||
return alarmService.findAlarmByIdAsync(tenantId, alarmId); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<AlarmInfo> findAlarmInfoByIdAsync(TenantId tenantId, AlarmId alarmId) { |
|||
return alarmService.findAlarmInfoByIdAsync(tenantId, alarmId); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<PageData<AlarmInfo>> findAlarms(TenantId tenantId, AlarmQuery query) { |
|||
return alarmService.findAlarms(tenantId, query); |
|||
} |
|||
|
|||
@Override |
|||
public AlarmSeverity findHighestAlarmSeverity(TenantId tenantId, EntityId entityId, AlarmSearchStatus alarmSearchStatus, AlarmStatus alarmStatus) { |
|||
return alarmService.findHighestAlarmSeverity(tenantId, entityId, alarmSearchStatus, alarmStatus); |
|||
} |
|||
|
|||
@Override |
|||
public PageData<AlarmData> findAlarmDataByQueryForEntities(TenantId tenantId, CustomerId customerId, AlarmDataQuery query, Collection<EntityId> orderedEntityIds) { |
|||
return alarmService.findAlarmDataByQueryForEntities(tenantId, customerId, query, orderedEntityIds); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Alarm> findLatestByOriginatorAndType(TenantId tenantId, EntityId originator, String type) { |
|||
return alarmService.findLatestByOriginatorAndType(tenantId, originator, type); |
|||
} |
|||
|
|||
private void onAlarmUpdated(AlarmOperationResult result) { |
|||
wsCallBackExecutor.submit(() -> { |
|||
Alarm alarm = result.getAlarm(); |
|||
TenantId tenantId = result.getAlarm().getTenantId(); |
|||
for (EntityId entityId : result.getPropagatedEntitiesList()) { |
|||
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); |
|||
if (currentPartitions.contains(tpi)) { |
|||
if (subscriptionManagerService.isPresent()) { |
|||
subscriptionManagerService.get().onAlarmUpdate(tenantId, entityId, alarm, TbCallback.EMPTY); |
|||
} else { |
|||
log.warn("Possible misconfiguration because subscriptionManagerService is null!"); |
|||
} |
|||
} else { |
|||
TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAlarmUpdateProto(tenantId, entityId, alarm); |
|||
clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); |
|||
} |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private void onAlarmDeleted(AlarmOperationResult result) { |
|||
wsCallBackExecutor.submit(() -> { |
|||
Alarm alarm = result.getAlarm(); |
|||
TenantId tenantId = result.getAlarm().getTenantId(); |
|||
for (EntityId entityId : result.getPropagatedEntitiesList()) { |
|||
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId); |
|||
if (currentPartitions.contains(tpi)) { |
|||
if (subscriptionManagerService.isPresent()) { |
|||
subscriptionManagerService.get().onAlarmDeleted(tenantId, entityId, alarm, TbCallback.EMPTY); |
|||
} else { |
|||
log.warn("Possible misconfiguration because subscriptionManagerService is null!"); |
|||
} |
|||
} else { |
|||
TransportProtos.ToCoreMsg toCoreMsg = TbSubscriptionUtils.toAlarmDeletedProto(tenantId, entityId, alarm); |
|||
clusterService.pushMsgToCore(tpi, entityId.getId(), toCoreMsg, null); |
|||
} |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private class AlarmUpdateCallback implements FutureCallback<AlarmOperationResult> { |
|||
@Override |
|||
public void onSuccess(@Nullable AlarmOperationResult result) { |
|||
onAlarmUpdated(result); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.warn("Failed to update alarm", t); |
|||
} |
|||
} |
|||
|
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue