committed by
GitHub
2843 changed files with 226669 additions and 166838 deletions
@ -0,0 +1,60 @@ |
|||
--- |
|||
name: "\U0001F41E Bug report" |
|||
about: Create a report to help us improve |
|||
title: "[Bug] " |
|||
labels: bug |
|||
assignees: ashvayka, vvlladd28 |
|||
|
|||
--- |
|||
|
|||
**Describe the bug** |
|||
A clear and concise description of what the bug is. |
|||
|
|||
**Your Server Environment** |
|||
<!-- 🔅🔅🔅🔅🔅🔅🔅 Choose one of the following or write your own 🔅🔅🔅🔅🔅🔅🔅--> |
|||
* demo.thingsboard.io |
|||
* cloud.thingsboard.io |
|||
* own setup |
|||
* cloud or local infrastructure or docker deployment |
|||
* ThingsBoard Version |
|||
* OS Name and Version |
|||
|
|||
**Your Client Environment** |
|||
<!-- 🔅🔅🔅🔅🔅🔅🔅 Choose one of the following or write your own 🔅🔅🔅🔅🔅🔅🔅--> |
|||
**Desktop (please complete the following information):** |
|||
|
|||
* OS: [e.g. iOS] |
|||
* Browser [e.g. chrome, safari] |
|||
* Version [e.g. 22] |
|||
|
|||
**Smartphone (please complete the following information):** |
|||
* Device: [e.g. iPhone6] |
|||
* OS: [e.g. iOS8.1] |
|||
* Browser [e.g. stock browser, safari] |
|||
* Version [e.g. 22] |
|||
|
|||
**Your Device** |
|||
|
|||
* Connectivity |
|||
* MQTT |
|||
* HTTP |
|||
* CoAP |
|||
* Gateway |
|||
* Integration: (Specify name) |
|||
* Device vendor and model |
|||
|
|||
**To Reproduce** |
|||
Steps to reproduce the behavior: |
|||
1. Go to '...' |
|||
2. Click on '....' |
|||
3. Scroll down to '....' |
|||
4. See error |
|||
|
|||
**Expected behavior** |
|||
A clear and concise description of what you expected to happen. |
|||
|
|||
**Screenshots** |
|||
If applicable, add screenshots to help explain your problem. |
|||
|
|||
**Additional context** |
|||
Add any other context about the problem here. |
|||
@ -0,0 +1,20 @@ |
|||
--- |
|||
name: Feature request |
|||
about: Suggest an idea for this project |
|||
title: "[Feature Request]" |
|||
labels: feature |
|||
assignees: ikulikov |
|||
|
|||
--- |
|||
|
|||
**Is your feature request related to a problem? Please describe.** |
|||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] |
|||
|
|||
**Describe the solution you'd like** |
|||
A clear and concise description of what you want to happen. |
|||
|
|||
**Describe alternatives you've considered** |
|||
A clear and concise description of any alternative solutions or features you've considered. |
|||
|
|||
**Additional context** |
|||
Add any other context or screenshots about the feature request here. |
|||
@ -0,0 +1,25 @@ |
|||
--- |
|||
name: Question |
|||
about: Describe your questions in details |
|||
title: "[Question] your title here" |
|||
labels: question |
|||
assignees: ashvayka |
|||
|
|||
--- |
|||
|
|||
**Component** |
|||
|
|||
<!-- Choose one of the following and delete all others. --> |
|||
* UI |
|||
* Rule Engine |
|||
* Installation |
|||
* Generic |
|||
|
|||
**Description** |
|||
A clear and concise details. |
|||
|
|||
**Environment** |
|||
<!-- Add information about your environment and ThingsBoard version if applicable --> |
|||
* OS: name and version |
|||
* ThingsBoard: version |
|||
* Browser: name and version |
|||
@ -1,12 +0,0 @@ |
|||
before_install: |
|||
- sudo rm -f /etc/mavenrc |
|||
- export M2_HOME=/usr/local/maven |
|||
- export MAVEN_OPTS="-Dmaven.repo.local=$HOME/.m2/repository -Xms1024m -Xmx3072m" |
|||
- export HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false |
|||
jdk: |
|||
- openjdk8 |
|||
language: java |
|||
sudo: required |
|||
services: |
|||
- docker |
|||
script: mvn clean verify -Ddockerfile.skip=false |
|||
@ -0,0 +1,23 @@ |
|||
{ |
|||
"providerId": "Facebook", |
|||
"accessTokenUri": "https://graph.facebook.com/v2.8/oauth/access_token", |
|||
"authorizationUri": "https://www.facebook.com/v2.8/dialog/oauth", |
|||
"scope": ["email","public_profile"], |
|||
"jwkSetUri": null, |
|||
"userInfoUri": "https://graph.facebook.com/me?fields=id,name,first_name,last_name,email", |
|||
"clientAuthenticationMethod": "BASIC", |
|||
"userNameAttributeName": "email", |
|||
"mapperConfig": { |
|||
"type": "BASIC", |
|||
"basic": { |
|||
"emailAttributeKey": "email", |
|||
"firstNameAttributeKey": "first_name", |
|||
"lastNameAttributeKey": "last_name", |
|||
"tenantNameStrategy": "DOMAIN" |
|||
} |
|||
}, |
|||
"comment": null, |
|||
"loginButtonIcon": "facebook-logo", |
|||
"loginButtonLabel": "Facebook", |
|||
"helpLink": "https://developers.facebook.com/docs/facebook-login/web#logindialog" |
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
{ |
|||
"providerId": "Github", |
|||
"accessTokenUri": "https://github.com/login/oauth/access_token", |
|||
"authorizationUri": "https://github.com/login/oauth/authorize", |
|||
"scope": ["read:user","user:email"], |
|||
"jwkSetUri": null, |
|||
"userInfoUri": "https://api.github.com/user", |
|||
"clientAuthenticationMethod": "BASIC", |
|||
"userNameAttributeName": "login", |
|||
"mapperConfig": { |
|||
"type": "GITHUB", |
|||
"basic": { |
|||
"firstNameAttributeKey": "name", |
|||
"tenantNameStrategy": "DOMAIN" |
|||
} |
|||
}, |
|||
"comment": "In order to log into ThingsBoard you need to have user's email. You may configure and use Custom OAuth2 Mapper to get email information. Please refer to <a href=\"https://docs.github.com/en/rest/reference/users#list-email-addresses-for-the-authenticated-user\">Github Documentation</a>", |
|||
"loginButtonIcon": "github-logo", |
|||
"loginButtonLabel": "Github", |
|||
"helpLink": "https://docs.github.com/en/developers/apps/creating-an-oauth-app" |
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
{ |
|||
"providerId": "Google", |
|||
"additionalInfo": null, |
|||
"accessTokenUri": "https://oauth2.googleapis.com/token", |
|||
"authorizationUri": "https://accounts.google.com/o/oauth2/v2/auth", |
|||
"scope": ["email","openid","profile"], |
|||
"jwkSetUri": "https://www.googleapis.com/oauth2/v3/certs", |
|||
"userInfoUri": "https://openidconnect.googleapis.com/v1/userinfo", |
|||
"clientAuthenticationMethod": "BASIC", |
|||
"userNameAttributeName": "email", |
|||
"mapperConfig": { |
|||
"type": "BASIC", |
|||
"basic": { |
|||
"emailAttributeKey": "email", |
|||
"firstNameAttributeKey": "given_name", |
|||
"lastNameAttributeKey": "family_name", |
|||
"tenantNameStrategy": "DOMAIN" |
|||
} |
|||
}, |
|||
"comment": null, |
|||
"loginButtonIcon": "google-logo", |
|||
"loginButtonLabel": "Google", |
|||
"helpLink": "https://developers.google.com/adwords/api/docs/guides/authentication" |
|||
} |
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,136 @@ |
|||
{ |
|||
"ruleChain": { |
|||
"additionalInfo": { |
|||
"description": "" |
|||
}, |
|||
"name": "Device Profile Rule Chain Template", |
|||
"firstRuleNodeId": null, |
|||
"root": false, |
|||
"debugMode": false, |
|||
"configuration": null |
|||
}, |
|||
"metadata": { |
|||
"firstNodeIndex": 6, |
|||
"nodes": [ |
|||
{ |
|||
"additionalInfo": { |
|||
"layoutX": 822, |
|||
"layoutY": 294 |
|||
}, |
|||
"type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", |
|||
"name": "Save Timeseries", |
|||
"debugMode": false, |
|||
"configuration": { |
|||
"defaultTTL": 0 |
|||
} |
|||
}, |
|||
{ |
|||
"additionalInfo": { |
|||
"layoutX": 824, |
|||
"layoutY": 221 |
|||
}, |
|||
"type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", |
|||
"name": "Save Client Attributes", |
|||
"debugMode": false, |
|||
"configuration": { |
|||
"scope": "CLIENT_SCOPE" |
|||
} |
|||
}, |
|||
{ |
|||
"additionalInfo": { |
|||
"layoutX": 494, |
|||
"layoutY": 309 |
|||
}, |
|||
"type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode", |
|||
"name": "Message Type Switch", |
|||
"debugMode": false, |
|||
"configuration": { |
|||
"version": 0 |
|||
} |
|||
}, |
|||
{ |
|||
"additionalInfo": { |
|||
"layoutX": 824, |
|||
"layoutY": 383 |
|||
}, |
|||
"type": "org.thingsboard.rule.engine.action.TbLogNode", |
|||
"name": "Log RPC from Device", |
|||
"debugMode": false, |
|||
"configuration": { |
|||
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" |
|||
} |
|||
}, |
|||
{ |
|||
"additionalInfo": { |
|||
"layoutX": 823, |
|||
"layoutY": 444 |
|||
}, |
|||
"type": "org.thingsboard.rule.engine.action.TbLogNode", |
|||
"name": "Log Other", |
|||
"debugMode": false, |
|||
"configuration": { |
|||
"jsScript": "return '\\nIncoming message:\\n' + JSON.stringify(msg) + '\\nIncoming metadata:\\n' + JSON.stringify(metadata);" |
|||
} |
|||
}, |
|||
{ |
|||
"additionalInfo": { |
|||
"layoutX": 822, |
|||
"layoutY": 507 |
|||
}, |
|||
"type": "org.thingsboard.rule.engine.rpc.TbSendRPCRequestNode", |
|||
"name": "RPC Call Request", |
|||
"debugMode": false, |
|||
"configuration": { |
|||
"timeoutInSeconds": 60 |
|||
} |
|||
}, |
|||
{ |
|||
"additionalInfo": { |
|||
"description": "", |
|||
"layoutX": 209, |
|||
"layoutY": 307 |
|||
}, |
|||
"type": "org.thingsboard.rule.engine.profile.TbDeviceProfileNode", |
|||
"name": "Device Profile Node", |
|||
"debugMode": false, |
|||
"configuration": { |
|||
"persistAlarmRulesState": false, |
|||
"fetchAlarmRulesStateOnStart": false |
|||
} |
|||
} |
|||
], |
|||
"connections": [ |
|||
{ |
|||
"fromIndex": 2, |
|||
"toIndex": 4, |
|||
"type": "Other" |
|||
}, |
|||
{ |
|||
"fromIndex": 2, |
|||
"toIndex": 1, |
|||
"type": "Post attributes" |
|||
}, |
|||
{ |
|||
"fromIndex": 2, |
|||
"toIndex": 0, |
|||
"type": "Post telemetry" |
|||
}, |
|||
{ |
|||
"fromIndex": 2, |
|||
"toIndex": 3, |
|||
"type": "RPC Request from Device" |
|||
}, |
|||
{ |
|||
"fromIndex": 2, |
|||
"toIndex": 5, |
|||
"type": "RPC Request to Device" |
|||
}, |
|||
{ |
|||
"fromIndex": 6, |
|||
"toIndex": 2, |
|||
"type": "Success" |
|||
} |
|||
], |
|||
"ruleChainConnections": null |
|||
} |
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
CREATE TABLE IF NOT EXISTS ts_kv_latest |
|||
( |
|||
entity_id uuid NOT NULL, |
|||
key int NOT NULL, |
|||
ts bigint NOT NULL, |
|||
bool_v boolean, |
|||
str_v varchar(10000000), |
|||
long_v bigint, |
|||
dbl_v double precision, |
|||
json_v json, |
|||
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS ts_kv_dictionary |
|||
( |
|||
key varchar(255) NOT NULL, |
|||
key_id serial UNIQUE, |
|||
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) |
|||
); |
|||
@ -0,0 +1,878 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS |
|||
$$ |
|||
BEGIN |
|||
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || |
|||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); |
|||
END; |
|||
$$ LANGUAGE plpgsql; |
|||
|
|||
CREATE OR REPLACE FUNCTION extract_ts(uuid UUID) RETURNS BIGINT AS |
|||
$$ |
|||
DECLARE |
|||
bytes bytea; |
|||
BEGIN |
|||
bytes := uuid_send(uuid); |
|||
RETURN |
|||
( |
|||
( |
|||
(get_byte(bytes, 0)::bigint << 24) | |
|||
(get_byte(bytes, 1)::bigint << 16) | |
|||
(get_byte(bytes, 2)::bigint << 8) | |
|||
(get_byte(bytes, 3)::bigint << 0) |
|||
) + ( |
|||
((get_byte(bytes, 4)::bigint << 8 | |
|||
get_byte(bytes, 5)::bigint)) << 32 |
|||
) + ( |
|||
(((get_byte(bytes, 6)::bigint & 15) << 8 | get_byte(bytes, 7)::bigint) & 4095) << 48 |
|||
) - 122192928000000000 |
|||
) / 10000::double precision; |
|||
END |
|||
$$ LANGUAGE plpgsql |
|||
IMMUTABLE |
|||
PARALLEL SAFE |
|||
RETURNS NULL ON NULL INPUT; |
|||
|
|||
|
|||
CREATE OR REPLACE FUNCTION column_type_to_uuid(table_name varchar, column_name varchar) RETURNS VOID |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
execute format('ALTER TABLE %s RENAME COLUMN %s TO old_%s;', table_name, column_name, column_name); |
|||
execute format('ALTER TABLE %s ADD COLUMN %s UUID;', table_name, column_name); |
|||
execute format('UPDATE %s SET %s = to_uuid(old_%s) WHERE old_%s is not null;', table_name, column_name, column_name, column_name); |
|||
execute format('ALTER TABLE %s DROP COLUMN old_%s;', table_name, column_name); |
|||
END; |
|||
$$; |
|||
|
|||
CREATE OR REPLACE FUNCTION get_column_type(table_name varchar, column_name varchar, OUT data_type varchar) RETURNS varchar |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
execute (format('SELECT data_type from information_schema.columns where table_name = %L and column_name = %L', |
|||
table_name, column_name)) INTO data_type; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
CREATE OR REPLACE PROCEDURE drop_all_idx() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
DROP INDEX IF EXISTS idx_alarm_originator_alarm_type; |
|||
DROP INDEX IF EXISTS idx_alarm_originator_created_time; |
|||
DROP INDEX IF EXISTS idx_alarm_tenant_created_time; |
|||
DROP INDEX IF EXISTS idx_event_type_entity_id; |
|||
DROP INDEX IF EXISTS idx_relation_to_id; |
|||
DROP INDEX IF EXISTS idx_relation_from_id; |
|||
DROP INDEX IF EXISTS idx_device_customer_id; |
|||
DROP INDEX IF EXISTS idx_device_customer_id_and_type; |
|||
DROP INDEX IF EXISTS idx_device_type; |
|||
DROP INDEX IF EXISTS idx_asset_customer_id; |
|||
DROP INDEX IF EXISTS idx_asset_customer_id_and_type; |
|||
DROP INDEX IF EXISTS idx_asset_type; |
|||
DROP INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts; |
|||
END; |
|||
$$; |
|||
|
|||
CREATE OR REPLACE PROCEDURE create_all_idx() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
CREATE INDEX IF NOT EXISTS idx_alarm_originator_alarm_type ON alarm(originator_id, type, start_ts DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_alarm_originator_created_time ON alarm(originator_id, created_time DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_alarm_tenant_created_time ON alarm(tenant_id, created_time DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_event_type_entity_id ON event(tenant_id, event_type, entity_type, entity_id); |
|||
CREATE INDEX IF NOT EXISTS idx_relation_to_id ON relation(relation_type_group, to_type, to_id); |
|||
CREATE INDEX IF NOT EXISTS idx_relation_from_id ON relation(relation_type_group, from_type, from_id); |
|||
CREATE INDEX IF NOT EXISTS idx_device_customer_id ON device(tenant_id, customer_id); |
|||
CREATE INDEX IF NOT EXISTS idx_device_customer_id_and_type ON device(tenant_id, customer_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_device_type ON device(tenant_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id); |
|||
CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); |
|||
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- admin_settings |
|||
CREATE OR REPLACE PROCEDURE update_admin_settings() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'admin_settings'; |
|||
column_id varchar := 'id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE admin_settings DROP CONSTRAINT admin_settings_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE admin_settings ADD CONSTRAINT admin_settings_pkey PRIMARY KEY (id); |
|||
ALTER TABLE admin_settings ADD COLUMN created_time BIGINT; |
|||
UPDATE admin_settings SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- alarm |
|||
CREATE OR REPLACE PROCEDURE update_alarm() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'alarm'; |
|||
column_id varchar := 'id'; |
|||
column_originator_id varchar := 'originator_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE alarm DROP CONSTRAINT alarm_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE alarm ADD COLUMN created_time BIGINT; |
|||
UPDATE alarm SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE alarm ADD CONSTRAINT alarm_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_originator_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_originator_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_originator_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_originator_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- asset |
|||
CREATE OR REPLACE PROCEDURE update_asset() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'asset'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE asset DROP CONSTRAINT asset_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE asset ADD COLUMN created_time BIGINT; |
|||
UPDATE asset SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE asset ADD CONSTRAINT asset_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE asset DROP CONSTRAINT asset_name_unq_key; |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
ALTER TABLE asset ADD CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- attribute_kv |
|||
CREATE OR REPLACE PROCEDURE update_attribute_kv() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'attribute_kv'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE attribute_kv DROP CONSTRAINT attribute_kv_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
ALTER TABLE attribute_kv ADD CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_type, entity_id, attribute_type, attribute_key); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- audit_log |
|||
CREATE OR REPLACE PROCEDURE update_audit_log() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'audit_log'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
column_user_id varchar := 'user_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE audit_log DROP CONSTRAINT audit_log_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE audit_log ADD COLUMN created_time BIGINT; |
|||
UPDATE audit_log SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE audit_log ADD CONSTRAINT audit_log_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_user_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_user_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_user_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_user_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- component_descriptor |
|||
CREATE OR REPLACE PROCEDURE update_component_descriptor() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'component_descriptor'; |
|||
column_id varchar := 'id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE component_descriptor DROP CONSTRAINT component_descriptor_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE component_descriptor ADD CONSTRAINT component_descriptor_pkey PRIMARY KEY (id); |
|||
ALTER TABLE component_descriptor ADD COLUMN created_time BIGINT; |
|||
UPDATE component_descriptor SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- customer |
|||
CREATE OR REPLACE PROCEDURE update_customer() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'customer'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE customer DROP CONSTRAINT customer_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE customer ADD CONSTRAINT customer_pkey PRIMARY KEY (id); |
|||
ALTER TABLE customer ADD COLUMN created_time BIGINT; |
|||
UPDATE customer SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- dashboard |
|||
CREATE OR REPLACE PROCEDURE update_dashboard() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'dashboard'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE dashboard DROP CONSTRAINT dashboard_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE dashboard ADD CONSTRAINT dashboard_pkey PRIMARY KEY (id); |
|||
ALTER TABLE dashboard ADD COLUMN created_time BIGINT; |
|||
UPDATE dashboard SET created_time = extract_ts(id) WHERE id is not null; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- device |
|||
CREATE OR REPLACE PROCEDURE update_device() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'device'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device DROP CONSTRAINT device_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE device ADD COLUMN created_time BIGINT; |
|||
UPDATE device SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE device ADD CONSTRAINT device_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device DROP CONSTRAINT device_name_unq_key; |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
ALTER TABLE device ADD CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- device_credentials |
|||
CREATE OR REPLACE PROCEDURE update_device_credentials() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'device_credentials'; |
|||
column_id varchar := 'id'; |
|||
column_device_id varchar := 'device_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device_credentials DROP CONSTRAINT device_credentials_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE device_credentials ADD COLUMN created_time BIGINT; |
|||
UPDATE device_credentials SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE device_credentials ADD CONSTRAINT device_credentials_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_device_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE device_credentials DROP CONSTRAINT IF EXISTS device_credentials_device_id_unq_key; |
|||
PERFORM column_type_to_uuid(table_name, column_device_id); |
|||
-- remove duplicate credentials with same device_id |
|||
DELETE from device_credentials where id in ( |
|||
select dc.id |
|||
from ( |
|||
SELECT id, device_id, |
|||
ROW_NUMBER() OVER ( |
|||
PARTITION BY |
|||
device_id |
|||
ORDER BY |
|||
created_time DESC |
|||
) row_num |
|||
FROM |
|||
device_credentials |
|||
) as dc |
|||
WHERE dc.row_num > 1 |
|||
); |
|||
ALTER TABLE device_credentials ADD CONSTRAINT device_credentials_device_id_unq_key UNIQUE (device_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_device_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_device_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- event |
|||
CREATE OR REPLACE PROCEDURE update_event() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'event'; |
|||
column_id varchar := 'id'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE event DROP CONSTRAINT event_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE event ADD COLUMN created_time BIGINT; |
|||
UPDATE event SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE event ADD CONSTRAINT event_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
ALTER TABLE event DROP CONSTRAINT event_unq_key; |
|||
|
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
|
|||
ALTER TABLE event ADD CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid); |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- relation |
|||
CREATE OR REPLACE PROCEDURE update_relation() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'relation'; |
|||
column_from_id varchar := 'from_id'; |
|||
column_to_id varchar := 'to_id'; |
|||
BEGIN |
|||
ALTER TABLE relation DROP CONSTRAINT relation_pkey; |
|||
|
|||
data_type := get_column_type(table_name, column_from_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_from_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_from_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_from_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_to_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_to_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_to_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_to_id; |
|||
END IF; |
|||
|
|||
ALTER TABLE relation ADD CONSTRAINT relation_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type); |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- tb_user |
|||
CREATE OR REPLACE PROCEDURE update_tb_user() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'tb_user'; |
|||
column_id varchar := 'id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE tb_user DROP CONSTRAINT tb_user_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE tb_user ADD COLUMN created_time BIGINT; |
|||
UPDATE tb_user SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE tb_user ADD CONSTRAINT tb_user_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- tenant |
|||
CREATE OR REPLACE PROCEDURE update_tenant() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'tenant'; |
|||
column_id varchar := 'id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE tenant DROP CONSTRAINT tenant_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE tenant ADD COLUMN created_time BIGINT; |
|||
UPDATE tenant SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE tenant ADD CONSTRAINT tenant_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- user_credentials |
|||
CREATE OR REPLACE PROCEDURE update_user_credentials() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'user_credentials'; |
|||
column_id varchar := 'id'; |
|||
column_user_id varchar := 'user_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE user_credentials DROP CONSTRAINT user_credentials_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE user_credentials ADD COLUMN created_time BIGINT; |
|||
UPDATE user_credentials SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE user_credentials ADD CONSTRAINT user_credentials_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_user_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE user_credentials DROP CONSTRAINT user_credentials_user_id_key; |
|||
ALTER TABLE user_credentials RENAME COLUMN user_id TO old_user_id; |
|||
ALTER TABLE user_credentials ADD COLUMN user_id UUID UNIQUE; |
|||
UPDATE user_credentials SET user_id = to_uuid(old_user_id) WHERE old_user_id is not null; |
|||
ALTER TABLE user_credentials DROP COLUMN old_user_id; |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_user_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_user_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- widget_type |
|||
CREATE OR REPLACE PROCEDURE update_widget_type() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'widget_type'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE widget_type DROP CONSTRAINT widget_type_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE widget_type ADD COLUMN created_time BIGINT; |
|||
UPDATE widget_type SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE widget_type ADD CONSTRAINT widget_type_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- widgets_bundle |
|||
CREATE OR REPLACE PROCEDURE update_widgets_bundle() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'widgets_bundle'; |
|||
column_id varchar := 'id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE widgets_bundle DROP CONSTRAINT widgets_bundle_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE widgets_bundle ADD COLUMN created_time BIGINT; |
|||
UPDATE widgets_bundle SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE widgets_bundle ADD CONSTRAINT widgets_bundle_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- rule_chain |
|||
CREATE OR REPLACE PROCEDURE update_rule_chain() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'rule_chain'; |
|||
column_id varchar := 'id'; |
|||
column_first_rule_node_id varchar := 'first_rule_node_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE rule_chain DROP CONSTRAINT rule_chain_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE rule_chain ADD COLUMN created_time BIGINT; |
|||
UPDATE rule_chain SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE rule_chain ADD CONSTRAINT rule_chain_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_first_rule_node_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_first_rule_node_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_first_rule_node_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_first_rule_node_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- rule_node |
|||
CREATE OR REPLACE PROCEDURE update_rule_node() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'rule_node'; |
|||
column_id varchar := 'id'; |
|||
column_rule_chain_id varchar := 'rule_chain_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE rule_node DROP CONSTRAINT rule_node_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE rule_node ADD COLUMN created_time BIGINT; |
|||
UPDATE rule_node SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE rule_node ADD CONSTRAINT rule_node_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_rule_chain_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_rule_chain_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_rule_chain_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_rule_chain_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
|
|||
-- entity_view |
|||
CREATE OR REPLACE PROCEDURE update_entity_view() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
data_type varchar; |
|||
table_name varchar := 'entity_view'; |
|||
column_id varchar := 'id'; |
|||
column_entity_id varchar := 'entity_id'; |
|||
column_tenant_id varchar := 'tenant_id'; |
|||
column_customer_id varchar := 'customer_id'; |
|||
BEGIN |
|||
data_type := get_column_type(table_name, column_id); |
|||
IF data_type = 'character varying' THEN |
|||
ALTER TABLE entity_view DROP CONSTRAINT entity_view_pkey; |
|||
PERFORM column_type_to_uuid(table_name, column_id); |
|||
ALTER TABLE entity_view ADD COLUMN created_time BIGINT; |
|||
UPDATE entity_view SET created_time = extract_ts(id) WHERE id is not null; |
|||
ALTER TABLE entity_view ADD CONSTRAINT entity_view_pkey PRIMARY KEY (id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_entity_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_entity_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_entity_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_entity_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_tenant_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_tenant_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_tenant_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_tenant_id; |
|||
END IF; |
|||
|
|||
data_type := get_column_type(table_name, column_customer_id); |
|||
IF data_type = 'character varying' THEN |
|||
PERFORM column_type_to_uuid(table_name, column_customer_id); |
|||
RAISE NOTICE 'Table % column % updated!', table_name, column_customer_id; |
|||
ELSE |
|||
RAISE NOTICE 'Table % column % already updated!', table_name, column_customer_id; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
CREATE TABLE IF NOT EXISTS ts_kv_latest |
|||
( |
|||
entity_id uuid NOT NULL, |
|||
key int NOT NULL, |
|||
ts bigint NOT NULL, |
|||
bool_v boolean, |
|||
str_v varchar(10000000), |
|||
long_v bigint, |
|||
dbl_v double precision, |
|||
json_v json, |
|||
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS ts_kv_dictionary |
|||
( |
|||
key varchar(255) NOT NULL, |
|||
key_id serial UNIQUE, |
|||
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) |
|||
); |
|||
@ -0,0 +1,17 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_alarm_tenant_alarm_type_created_time ON alarm(tenant_id, type, created_time DESC); |
|||
@ -0,0 +1,28 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
DROP PROCEDURE IF EXISTS update_tenant_profiles; |
|||
DROP PROCEDURE IF EXISTS update_device_profiles; |
|||
|
|||
ALTER TABLE tenant ALTER COLUMN tenant_profile_id SET NOT NULL; |
|||
ALTER TABLE tenant DROP CONSTRAINT IF EXISTS fk_tenant_profile; |
|||
ALTER TABLE tenant ADD CONSTRAINT fk_tenant_profile FOREIGN KEY (tenant_profile_id) REFERENCES tenant_profile(id); |
|||
ALTER TABLE tenant DROP COLUMN IF EXISTS isolated_tb_core; |
|||
ALTER TABLE tenant DROP COLUMN IF EXISTS isolated_tb_rule_engine; |
|||
|
|||
ALTER TABLE device ALTER COLUMN device_profile_id SET NOT NULL; |
|||
ALTER TABLE device DROP CONSTRAINT IF EXISTS fk_device_profile; |
|||
ALTER TABLE device ADD CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id); |
|||
@ -0,0 +1,154 @@ |
|||
-- |
|||
-- Copyright © 2016-2020 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
CREATE TABLE IF NOT EXISTS oauth2_client_registration_info ( |
|||
id uuid NOT NULL CONSTRAINT oauth2_client_registration_info_pkey PRIMARY KEY, |
|||
enabled boolean, |
|||
created_time bigint NOT NULL, |
|||
additional_info varchar, |
|||
client_id varchar(255), |
|||
client_secret varchar(255), |
|||
authorization_uri varchar(255), |
|||
token_uri varchar(255), |
|||
scope varchar(255), |
|||
user_info_uri varchar(255), |
|||
user_name_attribute_name varchar(255), |
|||
jwk_set_uri varchar(255), |
|||
client_authentication_method varchar(255), |
|||
login_button_label varchar(255), |
|||
login_button_icon varchar(255), |
|||
allow_user_creation boolean, |
|||
activate_user boolean, |
|||
type varchar(31), |
|||
basic_email_attribute_key varchar(31), |
|||
basic_first_name_attribute_key varchar(31), |
|||
basic_last_name_attribute_key varchar(31), |
|||
basic_tenant_name_strategy varchar(31), |
|||
basic_tenant_name_pattern varchar(255), |
|||
basic_customer_name_pattern varchar(255), |
|||
basic_default_dashboard_name varchar(255), |
|||
basic_always_full_screen boolean, |
|||
custom_url varchar(255), |
|||
custom_username varchar(255), |
|||
custom_password varchar(255), |
|||
custom_send_token boolean |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS oauth2_client_registration ( |
|||
id uuid NOT NULL CONSTRAINT oauth2_client_registration_pkey PRIMARY KEY, |
|||
created_time bigint NOT NULL, |
|||
domain_name varchar(255), |
|||
domain_scheme varchar(31), |
|||
client_registration_info_id uuid |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( |
|||
id uuid NOT NULL CONSTRAINT oauth2_client_registration_template_pkey PRIMARY KEY, |
|||
created_time bigint NOT NULL, |
|||
additional_info varchar, |
|||
provider_id varchar(255), |
|||
authorization_uri varchar(255), |
|||
token_uri varchar(255), |
|||
scope varchar(255), |
|||
user_info_uri varchar(255), |
|||
user_name_attribute_name varchar(255), |
|||
jwk_set_uri varchar(255), |
|||
client_authentication_method varchar(255), |
|||
type varchar(31), |
|||
basic_email_attribute_key varchar(31), |
|||
basic_first_name_attribute_key varchar(31), |
|||
basic_last_name_attribute_key varchar(31), |
|||
basic_tenant_name_strategy varchar(31), |
|||
basic_tenant_name_pattern varchar(255), |
|||
basic_customer_name_pattern varchar(255), |
|||
basic_default_dashboard_name varchar(255), |
|||
basic_always_full_screen boolean, |
|||
comment varchar, |
|||
login_button_icon varchar(255), |
|||
login_button_label varchar(255), |
|||
help_link varchar(255), |
|||
CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS device_profile ( |
|||
id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, |
|||
created_time bigint NOT NULL, |
|||
name varchar(255), |
|||
type varchar(255), |
|||
transport_type varchar(255), |
|||
provision_type varchar(255), |
|||
profile_data jsonb, |
|||
description varchar, |
|||
search_text varchar(255), |
|||
is_default boolean, |
|||
tenant_id uuid, |
|||
default_rule_chain_id uuid, |
|||
default_queue_name varchar(255), |
|||
provision_device_key varchar, |
|||
CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), |
|||
CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), |
|||
CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id) |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS tenant_profile ( |
|||
id uuid NOT NULL CONSTRAINT tenant_profile_pkey PRIMARY KEY, |
|||
created_time bigint NOT NULL, |
|||
name varchar(255), |
|||
profile_data jsonb, |
|||
description varchar, |
|||
search_text varchar(255), |
|||
is_default boolean, |
|||
isolated_tb_core boolean, |
|||
isolated_tb_rule_engine boolean, |
|||
CONSTRAINT tenant_profile_name_unq_key UNIQUE (name) |
|||
); |
|||
|
|||
CREATE OR REPLACE PROCEDURE update_tenant_profiles() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
UPDATE tenant as t SET tenant_profile_id = p.id |
|||
FROM |
|||
(SELECT id from tenant_profile WHERE isolated_tb_core = false AND isolated_tb_rule_engine = false) as p |
|||
WHERE t.tenant_profile_id IS NULL AND t.isolated_tb_core = false AND t.isolated_tb_rule_engine = false; |
|||
|
|||
UPDATE tenant as t SET tenant_profile_id = p.id |
|||
FROM |
|||
(SELECT id from tenant_profile WHERE isolated_tb_core = true AND isolated_tb_rule_engine = false) as p |
|||
WHERE t.tenant_profile_id IS NULL AND t.isolated_tb_core = true AND t.isolated_tb_rule_engine = false; |
|||
|
|||
UPDATE tenant as t SET tenant_profile_id = p.id |
|||
FROM |
|||
(SELECT id from tenant_profile WHERE isolated_tb_core = false AND isolated_tb_rule_engine = true) as p |
|||
WHERE t.tenant_profile_id IS NULL AND t.isolated_tb_core = false AND t.isolated_tb_rule_engine = true; |
|||
|
|||
UPDATE tenant as t SET tenant_profile_id = p.id |
|||
FROM |
|||
(SELECT id from tenant_profile WHERE isolated_tb_core = true AND isolated_tb_rule_engine = true) as p |
|||
WHERE t.tenant_profile_id IS NULL AND t.isolated_tb_core = true AND t.isolated_tb_rule_engine = true; |
|||
END; |
|||
$$; |
|||
|
|||
CREATE OR REPLACE PROCEDURE update_device_profiles() |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
BEGIN |
|||
UPDATE device as d SET device_profile_id = p.id, device_data = '{"configuration":{"type":"DEFAULT"}, "transportConfiguration":{"type":"DEFAULT"}}' |
|||
FROM |
|||
(SELECT id, tenant_id, name from device_profile) as p |
|||
WHERE d.device_profile_id IS NULL AND p.tenant_id = d.tenant_id AND d.type = p.name; |
|||
END; |
|||
$$; |
|||
@ -0,0 +1,267 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.config; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.security.crypto.keygen.Base64StringKeyGenerator; |
|||
import org.springframework.security.crypto.keygen.StringKeyGenerator; |
|||
import org.springframework.security.oauth2.client.registration.ClientRegistration; |
|||
import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; |
|||
import org.springframework.security.oauth2.client.web.OAuth2AuthorizationRequestResolver; |
|||
import org.springframework.security.oauth2.core.AuthorizationGrantType; |
|||
import org.springframework.security.oauth2.core.ClientAuthenticationMethod; |
|||
import org.springframework.security.oauth2.core.endpoint.OAuth2AuthorizationRequest; |
|||
import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; |
|||
import org.springframework.security.oauth2.core.endpoint.PkceParameterNames; |
|||
import org.springframework.security.oauth2.core.oidc.OidcScopes; |
|||
import org.springframework.security.oauth2.core.oidc.endpoint.OidcParameterNames; |
|||
import org.springframework.security.web.util.UrlUtils; |
|||
import org.springframework.security.web.util.matcher.AntPathRequestMatcher; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.util.CollectionUtils; |
|||
import org.springframework.util.StringUtils; |
|||
import org.springframework.web.util.UriComponents; |
|||
import org.springframework.web.util.UriComponentsBuilder; |
|||
import org.thingsboard.server.dao.oauth2.OAuth2Configuration; |
|||
import org.thingsboard.server.utils.MiscUtils; |
|||
|
|||
import javax.servlet.http.HttpServletRequest; |
|||
import java.nio.charset.StandardCharsets; |
|||
import java.security.MessageDigest; |
|||
import java.security.NoSuchAlgorithmException; |
|||
import java.util.Base64; |
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
public class CustomOAuth2AuthorizationRequestResolver implements OAuth2AuthorizationRequestResolver { |
|||
public static final String DEFAULT_AUTHORIZATION_REQUEST_BASE_URI = "/oauth2/authorization"; |
|||
public static final String DEFAULT_LOGIN_PROCESSING_URI = "/login/oauth2/code/"; |
|||
private static final String REGISTRATION_ID_URI_VARIABLE_NAME = "registrationId"; |
|||
private static final char PATH_DELIMITER = '/'; |
|||
|
|||
private final AntPathRequestMatcher authorizationRequestMatcher = new AntPathRequestMatcher( |
|||
DEFAULT_AUTHORIZATION_REQUEST_BASE_URI + "/{" + REGISTRATION_ID_URI_VARIABLE_NAME + "}"); |
|||
private final StringKeyGenerator stateGenerator = new Base64StringKeyGenerator(Base64.getUrlEncoder()); |
|||
private final StringKeyGenerator secureKeyGenerator = new Base64StringKeyGenerator(Base64.getUrlEncoder().withoutPadding(), 96); |
|||
|
|||
@Autowired |
|||
private ClientRegistrationRepository clientRegistrationRepository; |
|||
|
|||
@Autowired(required = false) |
|||
private OAuth2Configuration oauth2Configuration; |
|||
|
|||
|
|||
@Override |
|||
public OAuth2AuthorizationRequest resolve(HttpServletRequest request) { |
|||
String registrationId = this.resolveRegistrationId(request); |
|||
String redirectUriAction = getAction(request, "login"); |
|||
return resolve(request, registrationId, redirectUriAction); |
|||
} |
|||
|
|||
@Override |
|||
public OAuth2AuthorizationRequest resolve(HttpServletRequest request, String registrationId) { |
|||
if (registrationId == null) { |
|||
return null; |
|||
} |
|||
String redirectUriAction = getAction(request, "authorize"); |
|||
return resolve(request, registrationId, redirectUriAction); |
|||
} |
|||
|
|||
private String getAction(HttpServletRequest request, String defaultAction) { |
|||
String action = request.getParameter("action"); |
|||
if (action == null) { |
|||
return defaultAction; |
|||
} |
|||
return action; |
|||
} |
|||
|
|||
private OAuth2AuthorizationRequest resolve(HttpServletRequest request, String registrationId, String redirectUriAction) { |
|||
if (registrationId == null) { |
|||
return null; |
|||
} |
|||
|
|||
ClientRegistration clientRegistration = this.clientRegistrationRepository.findByRegistrationId(registrationId); |
|||
if (clientRegistration == null) { |
|||
throw new IllegalArgumentException("Invalid Client Registration with Id: " + registrationId); |
|||
} |
|||
|
|||
Map<String, Object> attributes = new HashMap<>(); |
|||
attributes.put(OAuth2ParameterNames.REGISTRATION_ID, clientRegistration.getRegistrationId()); |
|||
|
|||
OAuth2AuthorizationRequest.Builder builder; |
|||
if (AuthorizationGrantType.AUTHORIZATION_CODE.equals(clientRegistration.getAuthorizationGrantType())) { |
|||
builder = OAuth2AuthorizationRequest.authorizationCode(); |
|||
Map<String, Object> additionalParameters = new HashMap<>(); |
|||
if (!CollectionUtils.isEmpty(clientRegistration.getScopes()) && |
|||
clientRegistration.getScopes().contains(OidcScopes.OPENID)) { |
|||
// Section 3.1.2.1 Authentication Request - https://openid.net/specs/openid-connect-core-1_0.html#AuthRequest
|
|||
// scope
|
|||
// REQUIRED. OpenID Connect requests MUST contain the "openid" scope value.
|
|||
addNonceParameters(attributes, additionalParameters); |
|||
} |
|||
if (ClientAuthenticationMethod.NONE.equals(clientRegistration.getClientAuthenticationMethod())) { |
|||
addPkceParameters(attributes, additionalParameters); |
|||
} |
|||
builder.additionalParameters(additionalParameters); |
|||
} else if (AuthorizationGrantType.IMPLICIT.equals(clientRegistration.getAuthorizationGrantType())) { |
|||
builder = OAuth2AuthorizationRequest.implicit(); |
|||
} else { |
|||
throw new IllegalArgumentException("Invalid Authorization Grant Type (" + |
|||
clientRegistration.getAuthorizationGrantType().getValue() + |
|||
") for Client Registration with Id: " + clientRegistration.getRegistrationId()); |
|||
} |
|||
|
|||
String redirectUriStr = expandRedirectUri(request, clientRegistration, redirectUriAction); |
|||
|
|||
return builder |
|||
.clientId(clientRegistration.getClientId()) |
|||
.authorizationUri(clientRegistration.getProviderDetails().getAuthorizationUri()) |
|||
.redirectUri(redirectUriStr) |
|||
.scopes(clientRegistration.getScopes()) |
|||
.state(this.stateGenerator.generateKey()) |
|||
.attributes(attributes) |
|||
.build(); |
|||
} |
|||
|
|||
private String resolveRegistrationId(HttpServletRequest request) { |
|||
if (this.authorizationRequestMatcher.matches(request)) { |
|||
return this.authorizationRequestMatcher |
|||
.matcher(request).getVariables().get(REGISTRATION_ID_URI_VARIABLE_NAME); |
|||
} |
|||
return null; |
|||
} |
|||
|
|||
/** |
|||
* Expands the {@link ClientRegistration#getRedirectUriTemplate()} with following provided variables:<br/> |
|||
* - baseUrl (e.g. https://localhost/app) <br/>
|
|||
* - baseScheme (e.g. https) <br/> |
|||
* - baseHost (e.g. localhost) <br/> |
|||
* - basePort (e.g. :8080) <br/> |
|||
* - basePath (e.g. /app) <br/> |
|||
* - registrationId (e.g. google) <br/> |
|||
* - action (e.g. login) <br/> |
|||
* <p/> |
|||
* Null variables are provided as empty strings. |
|||
* <p/> |
|||
* Default redirectUriTemplate is: {@link org.springframework.security.config.oauth2.client}.CommonOAuth2Provider#DEFAULT_REDIRECT_URL |
|||
* |
|||
* @return expanded URI |
|||
*/ |
|||
private String expandRedirectUri(HttpServletRequest request, ClientRegistration clientRegistration, String action) { |
|||
Map<String, String> uriVariables = new HashMap<>(); |
|||
uriVariables.put("registrationId", clientRegistration.getRegistrationId()); |
|||
|
|||
UriComponents uriComponents = UriComponentsBuilder.fromHttpUrl(UrlUtils.buildFullRequestUrl(request)) |
|||
.replacePath(request.getContextPath()) |
|||
.replaceQuery(null) |
|||
.fragment(null) |
|||
.build(); |
|||
String scheme = uriComponents.getScheme(); |
|||
uriVariables.put("baseScheme", scheme == null ? "" : scheme); |
|||
String host = uriComponents.getHost(); |
|||
uriVariables.put("baseHost", host == null ? "" : host); |
|||
// following logic is based on HierarchicalUriComponents#toUriString()
|
|||
int port = uriComponents.getPort(); |
|||
uriVariables.put("basePort", port == -1 ? "" : ":" + port); |
|||
String path = uriComponents.getPath(); |
|||
if (StringUtils.hasLength(path)) { |
|||
if (path.charAt(0) != PATH_DELIMITER) { |
|||
path = PATH_DELIMITER + path; |
|||
} |
|||
} |
|||
uriVariables.put("basePath", path == null ? "" : path); |
|||
uriVariables.put("baseUrl", uriComponents.toUriString()); |
|||
|
|||
uriVariables.put("action", action == null ? "" : action); |
|||
|
|||
String redirectUri = getRedirectUri(request); |
|||
log.trace("Redirect URI - {}.", redirectUri); |
|||
|
|||
return UriComponentsBuilder.fromUriString(redirectUri) |
|||
.buildAndExpand(uriVariables) |
|||
.toUriString(); |
|||
} |
|||
|
|||
private String getRedirectUri(HttpServletRequest request) { |
|||
String loginProcessingUri = oauth2Configuration != null ? oauth2Configuration.getLoginProcessingUrl() : DEFAULT_LOGIN_PROCESSING_URI; |
|||
|
|||
String scheme = MiscUtils.getScheme(request); |
|||
String domainName = MiscUtils.getDomainName(request); |
|||
int port = MiscUtils.getPort(request); |
|||
String baseUrl = scheme + "://" + domainName; |
|||
if (needsPort(scheme, port)){ |
|||
baseUrl += ":" + port; |
|||
} |
|||
return baseUrl + loginProcessingUri; |
|||
} |
|||
|
|||
private boolean needsPort(String scheme, int port) { |
|||
boolean isHttpDefault = "http".equals(scheme.toLowerCase()) && port == 80; |
|||
boolean isHttpsDefault = "https".equals(scheme.toLowerCase()) && port == 443; |
|||
return !isHttpDefault && !isHttpsDefault; |
|||
} |
|||
|
|||
/** |
|||
* Creates nonce and its hash for use in OpenID Connect 1.0 Authentication Requests. |
|||
* |
|||
* @param attributes where the {@link OidcParameterNames#NONCE} is stored for the authentication request |
|||
* @param additionalParameters where the {@link OidcParameterNames#NONCE} hash is added for the authentication request |
|||
* |
|||
* @since 5.2 |
|||
* @see <a target="_blank" href="https://openid.net/specs/openid-connect-core-1_0.html#AuthRequest">3.1.2.1. Authentication Request</a> |
|||
*/ |
|||
private void addNonceParameters(Map<String, Object> attributes, Map<String, Object> additionalParameters) { |
|||
try { |
|||
String nonce = this.secureKeyGenerator.generateKey(); |
|||
String nonceHash = createHash(nonce); |
|||
attributes.put(OidcParameterNames.NONCE, nonce); |
|||
additionalParameters.put(OidcParameterNames.NONCE, nonceHash); |
|||
} catch (NoSuchAlgorithmException e) { } |
|||
} |
|||
|
|||
/** |
|||
* Creates and adds additional PKCE parameters for use in the OAuth 2.0 Authorization and Access Token Requests |
|||
* |
|||
* @param attributes where {@link PkceParameterNames#CODE_VERIFIER} is stored for the token request |
|||
* @param additionalParameters where {@link PkceParameterNames#CODE_CHALLENGE} and, usually, |
|||
* {@link PkceParameterNames#CODE_CHALLENGE_METHOD} are added to be used in the authorization request. |
|||
* |
|||
* @since 5.2 |
|||
* @see <a target="_blank" href="https://tools.ietf.org/html/rfc7636#section-1.1">1.1. Protocol Flow</a> |
|||
* @see <a target="_blank" href="https://tools.ietf.org/html/rfc7636#section-4.1">4.1. Client Creates a Code Verifier</a> |
|||
* @see <a target="_blank" href="https://tools.ietf.org/html/rfc7636#section-4.2">4.2. Client Creates the Code Challenge</a> |
|||
*/ |
|||
private void addPkceParameters(Map<String, Object> attributes, Map<String, Object> additionalParameters) { |
|||
String codeVerifier = this.secureKeyGenerator.generateKey(); |
|||
attributes.put(PkceParameterNames.CODE_VERIFIER, codeVerifier); |
|||
try { |
|||
String codeChallenge = createHash(codeVerifier); |
|||
additionalParameters.put(PkceParameterNames.CODE_CHALLENGE, codeChallenge); |
|||
additionalParameters.put(PkceParameterNames.CODE_CHALLENGE_METHOD, "S256"); |
|||
} catch (NoSuchAlgorithmException e) { |
|||
additionalParameters.put(PkceParameterNames.CODE_CHALLENGE, codeVerifier); |
|||
} |
|||
} |
|||
|
|||
private static String createHash(String value) throws NoSuchAlgorithmException { |
|||
MessageDigest md = MessageDigest.getInstance("SHA-256"); |
|||
byte[] digest = md.digest(value.getBytes(StandardCharsets.US_ASCII)); |
|||
return Base64.getUrlEncoder().withoutPadding().encodeToString(digest); |
|||
} |
|||
} |
|||
@ -0,0 +1,202 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.http.HttpStatus; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.PathVariable; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RequestMethod; |
|||
import org.springframework.web.bind.annotation.RequestParam; |
|||
import org.springframework.web.bind.annotation.ResponseBody; |
|||
import org.springframework.web.bind.annotation.ResponseStatus; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
import org.thingsboard.server.common.data.DeviceProfile; |
|||
import org.thingsboard.server.common.data.DeviceProfileInfo; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.audit.ActionType; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.DeviceProfileId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.security.permission.Operation; |
|||
import org.thingsboard.server.service.security.permission.Resource; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api") |
|||
@Slf4j |
|||
public class DeviceProfileController extends BaseController { |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/deviceProfile/{deviceProfileId}", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public DeviceProfile getDeviceProfileById(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException { |
|||
checkParameter("deviceProfileId", strDeviceProfileId); |
|||
try { |
|||
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId)); |
|||
return checkDeviceProfileId(deviceProfileId, Operation.READ); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/deviceProfileInfo/{deviceProfileId}", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public DeviceProfileInfo getDeviceProfileInfoById(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException { |
|||
checkParameter("deviceProfileId", strDeviceProfileId); |
|||
try { |
|||
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId)); |
|||
return checkNotNull(deviceProfileService.findDeviceProfileInfoById(getTenantId(), deviceProfileId)); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/deviceProfileInfo/default", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public DeviceProfileInfo getDefaultDeviceProfileInfo() throws ThingsboardException { |
|||
try { |
|||
return checkNotNull(deviceProfileService.findDefaultDeviceProfileInfo(getTenantId())); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/deviceProfile", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public DeviceProfile saveDeviceProfile(@RequestBody DeviceProfile deviceProfile) throws ThingsboardException { |
|||
try { |
|||
boolean created = deviceProfile.getId() == null; |
|||
deviceProfile.setTenantId(getTenantId()); |
|||
|
|||
checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE); |
|||
|
|||
DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); |
|||
|
|||
tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); |
|||
tbClusterService.onEntityStateChange(deviceProfile.getTenantId(), savedDeviceProfile.getId(), |
|||
created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); |
|||
|
|||
logEntityAction(savedDeviceProfile.getId(), savedDeviceProfile, |
|||
null, |
|||
savedDeviceProfile.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); |
|||
|
|||
return savedDeviceProfile; |
|||
} catch (Exception e) { |
|||
logEntityAction(emptyId(EntityType.DEVICE_PROFILE), deviceProfile, |
|||
null, deviceProfile.getId() == null ? ActionType.ADDED : ActionType.UPDATED, e); |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/deviceProfile/{deviceProfileId}", method = RequestMethod.DELETE) |
|||
@ResponseStatus(value = HttpStatus.OK) |
|||
public void deleteDeviceProfile(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException { |
|||
checkParameter("deviceProfileId", strDeviceProfileId); |
|||
try { |
|||
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId)); |
|||
DeviceProfile deviceProfile = checkDeviceProfileId(deviceProfileId, Operation.DELETE); |
|||
deviceProfileService.deleteDeviceProfile(getTenantId(), deviceProfileId); |
|||
|
|||
tbClusterService.onDeviceProfileDelete(deviceProfile, null); |
|||
tbClusterService.onEntityStateChange(deviceProfile.getTenantId(), deviceProfile.getId(), ComponentLifecycleEvent.DELETED); |
|||
|
|||
logEntityAction(deviceProfileId, deviceProfile, |
|||
null, |
|||
ActionType.DELETED, null, strDeviceProfileId); |
|||
|
|||
} catch (Exception e) { |
|||
logEntityAction(emptyId(EntityType.DEVICE_PROFILE), |
|||
null, |
|||
null, |
|||
ActionType.DELETED, e, strDeviceProfileId); |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/deviceProfile/{deviceProfileId}/default", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public DeviceProfile setDefaultDeviceProfile(@PathVariable("deviceProfileId") String strDeviceProfileId) throws ThingsboardException { |
|||
checkParameter("deviceProfileId", strDeviceProfileId); |
|||
try { |
|||
DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId)); |
|||
DeviceProfile deviceProfile = checkDeviceProfileId(deviceProfileId, Operation.WRITE); |
|||
DeviceProfile previousDefaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(getTenantId()); |
|||
if (deviceProfileService.setDefaultDeviceProfile(getTenantId(), deviceProfileId)) { |
|||
if (previousDefaultDeviceProfile != null) { |
|||
previousDefaultDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), previousDefaultDeviceProfile.getId()); |
|||
|
|||
logEntityAction(previousDefaultDeviceProfile.getId(), previousDefaultDeviceProfile, |
|||
null, ActionType.UPDATED, null); |
|||
} |
|||
deviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfileId); |
|||
|
|||
logEntityAction(deviceProfile.getId(), deviceProfile, |
|||
null, ActionType.UPDATED, null); |
|||
} |
|||
return deviceProfile; |
|||
} catch (Exception e) { |
|||
logEntityAction(emptyId(EntityType.DEVICE_PROFILE), |
|||
null, |
|||
null, |
|||
ActionType.UPDATED, e, strDeviceProfileId); |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/deviceProfiles", params = {"pageSize", "page"}, method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public PageData<DeviceProfile> getDeviceProfiles(@RequestParam int pageSize, |
|||
@RequestParam int page, |
|||
@RequestParam(required = false) String textSearch, |
|||
@RequestParam(required = false) String sortProperty, |
|||
@RequestParam(required = false) String sortOrder) throws ThingsboardException { |
|||
try { |
|||
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); |
|||
return checkNotNull(deviceProfileService.findDeviceProfiles(getTenantId(), pageLink)); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/deviceProfileInfos", params = {"pageSize", "page"}, method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public PageData<DeviceProfileInfo> getDeviceProfileInfos(@RequestParam int pageSize, |
|||
@RequestParam int page, |
|||
@RequestParam(required = false) String textSearch, |
|||
@RequestParam(required = false) String sortProperty, |
|||
@RequestParam(required = false) String sortOrder, |
|||
@RequestParam(required = false) String transportType) throws ThingsboardException { |
|||
try { |
|||
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); |
|||
return checkNotNull(deviceProfileService.findDeviceProfileInfos(getTenantId(), pageLink, transportType)); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,79 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RequestMethod; |
|||
import org.springframework.web.bind.annotation.ResponseBody; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.query.AlarmData; |
|||
import org.thingsboard.server.common.data.query.AlarmDataQuery; |
|||
import org.thingsboard.server.common.data.query.EntityCountQuery; |
|||
import org.thingsboard.server.common.data.query.EntityData; |
|||
import org.thingsboard.server.common.data.query.EntityDataQuery; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.query.EntityQueryService; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api") |
|||
public class EntityQueryController extends BaseController { |
|||
|
|||
@Autowired |
|||
private EntityQueryService entityQueryService; |
|||
|
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/entitiesQuery/count", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public long countEntitiesByQuery(@RequestBody EntityCountQuery query) throws ThingsboardException { |
|||
checkNotNull(query); |
|||
try { |
|||
return this.entityQueryService.countEntitiesByQuery(getCurrentUser(), query); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/entitiesQuery/find", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public PageData<EntityData> findEntityDataByQuery(@RequestBody EntityDataQuery query) throws ThingsboardException { |
|||
checkNotNull(query); |
|||
try { |
|||
return this.entityQueryService.findEntityDataByQuery(getCurrentUser(), query); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") |
|||
@RequestMapping(value = "/alarmsQuery/find", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public PageData<AlarmData> findAlarmDataByQuery(@RequestBody AlarmDataQuery query) throws ThingsboardException { |
|||
checkNotNull(query); |
|||
try { |
|||
return this.entityQueryService.findAlarmDataByQuery(getCurrentUser(), query); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,77 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.http.HttpStatus; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.*; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.audit.ActionType; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationTemplateId; |
|||
import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.security.permission.Operation; |
|||
import org.thingsboard.server.service.security.permission.Resource; |
|||
|
|||
import java.util.List; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api/oauth2/config/template") |
|||
@Slf4j |
|||
public class OAuth2ConfigTemplateController extends BaseController { |
|||
private static final String CLIENT_REGISTRATION_TEMPLATE_ID = "clientRegistrationTemplateId"; |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(method = RequestMethod.POST) |
|||
@ResponseStatus(value = HttpStatus.OK) |
|||
public OAuth2ClientRegistrationTemplate saveClientRegistrationTemplate(@RequestBody OAuth2ClientRegistrationTemplate clientRegistrationTemplate) throws ThingsboardException { |
|||
try { |
|||
accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_TEMPLATE, Operation.WRITE); |
|||
return oAuth2ConfigTemplateService.saveClientRegistrationTemplate(clientRegistrationTemplate); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/{clientRegistrationTemplateId}", method = RequestMethod.DELETE) |
|||
@ResponseStatus(value = HttpStatus.OK) |
|||
public void deleteClientRegistrationTemplate(@PathVariable(CLIENT_REGISTRATION_TEMPLATE_ID) String strClientRegistrationTemplateId) throws ThingsboardException { |
|||
checkParameter(CLIENT_REGISTRATION_TEMPLATE_ID, strClientRegistrationTemplateId); |
|||
try { |
|||
accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_TEMPLATE, Operation.DELETE); |
|||
OAuth2ClientRegistrationTemplateId clientRegistrationTemplateId = new OAuth2ClientRegistrationTemplateId(toUUID(strClientRegistrationTemplateId)); |
|||
oAuth2ConfigTemplateService.deleteClientRegistrationTemplateById(clientRegistrationTemplateId); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") |
|||
@RequestMapping(method = RequestMethod.GET, produces = "application/json") |
|||
@ResponseBody |
|||
public List<OAuth2ClientRegistrationTemplate> getClientRegistrationTemplates() throws ThingsboardException { |
|||
try { |
|||
accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_TEMPLATE, Operation.READ); |
|||
return oAuth2ConfigTemplateService.findAllClientRegistrationTemplates(); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,91 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.http.HttpStatus; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.*; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; |
|||
import org.thingsboard.server.common.data.oauth2.OAuth2ClientsParams; |
|||
import org.thingsboard.server.common.data.oauth2.SchemeType; |
|||
import org.thingsboard.server.dao.oauth2.OAuth2Configuration; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.security.permission.Operation; |
|||
import org.thingsboard.server.service.security.permission.Resource; |
|||
import org.thingsboard.server.utils.MiscUtils; |
|||
|
|||
import javax.servlet.http.HttpServletRequest; |
|||
import java.util.List; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api") |
|||
@Slf4j |
|||
public class OAuth2Controller extends BaseController { |
|||
|
|||
@Autowired |
|||
private OAuth2Configuration oAuth2Configuration; |
|||
|
|||
@RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public List<OAuth2ClientInfo> getOAuth2Clients(HttpServletRequest request) throws ThingsboardException { |
|||
try { |
|||
return oAuth2Service.getOAuth2Clients(MiscUtils.getScheme(request), MiscUtils.getDomainNameAndPort(request)); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/oauth2/config", method = RequestMethod.GET, produces = "application/json") |
|||
@ResponseBody |
|||
public OAuth2ClientsParams getCurrentOAuth2Params() throws ThingsboardException { |
|||
try { |
|||
accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.READ); |
|||
return oAuth2Service.findOAuth2Params(); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/oauth2/config", method = RequestMethod.POST) |
|||
@ResponseStatus(value = HttpStatus.OK) |
|||
public OAuth2ClientsParams saveOAuth2Params(@RequestBody OAuth2ClientsParams oauth2Params) throws ThingsboardException { |
|||
try { |
|||
accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.WRITE); |
|||
oAuth2Service.saveOAuth2Params(oauth2Params); |
|||
return oAuth2Service.findOAuth2Params(); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/oauth2/loginProcessingUrl", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public String getLoginProcessingUrl() throws ThingsboardException { |
|||
try { |
|||
accessControlService.checkPermission(getCurrentUser(), Resource.OAUTH2_CONFIGURATION_INFO, Operation.READ); |
|||
return "\"" + oAuth2Configuration.getLoginProcessingUrl() + "\""; |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,170 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.http.HttpStatus; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.PathVariable; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RequestMethod; |
|||
import org.springframework.web.bind.annotation.RequestParam; |
|||
import org.springframework.web.bind.annotation.ResponseBody; |
|||
import org.springframework.web.bind.annotation.ResponseStatus; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
import org.thingsboard.server.common.data.EntityInfo; |
|||
import org.thingsboard.server.common.data.TenantProfile; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.TenantProfileId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; |
|||
import org.thingsboard.server.dao.exception.DataValidationException; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.security.permission.Operation; |
|||
import org.thingsboard.server.service.security.permission.Resource; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api") |
|||
@Slf4j |
|||
public class TenantProfileController extends BaseController { |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfile/{tenantProfileId}", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public TenantProfile getTenantProfileById(@PathVariable("tenantProfileId") String strTenantProfileId) throws ThingsboardException { |
|||
checkParameter("tenantProfileId", strTenantProfileId); |
|||
try { |
|||
TenantProfileId tenantProfileId = new TenantProfileId(toUUID(strTenantProfileId)); |
|||
return checkTenantProfileId(tenantProfileId, Operation.READ); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfileInfo/{tenantProfileId}", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public EntityInfo getTenantProfileInfoById(@PathVariable("tenantProfileId") String strTenantProfileId) throws ThingsboardException { |
|||
checkParameter("tenantProfileId", strTenantProfileId); |
|||
try { |
|||
TenantProfileId tenantProfileId = new TenantProfileId(toUUID(strTenantProfileId)); |
|||
return checkNotNull(tenantProfileService.findTenantProfileInfoById(getTenantId(), tenantProfileId)); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfileInfo/default", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public EntityInfo getDefaultTenantProfileInfo() throws ThingsboardException { |
|||
try { |
|||
return checkNotNull(tenantProfileService.findDefaultTenantProfileInfo(getTenantId())); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfile", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public TenantProfile saveTenantProfile(@RequestBody TenantProfile tenantProfile) throws ThingsboardException { |
|||
try { |
|||
boolean newTenantProfile = tenantProfile.getId() == null; |
|||
if (newTenantProfile) { |
|||
accessControlService |
|||
.checkPermission(getCurrentUser(), Resource.TENANT_PROFILE, Operation.CREATE); |
|||
} else { |
|||
checkEntityId(tenantProfile.getId(), Operation.WRITE); |
|||
} |
|||
|
|||
tenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(getTenantId(), tenantProfile)); |
|||
tenantProfileCache.put(tenantProfile); |
|||
tbClusterService.onTenantProfileChange(tenantProfile, null); |
|||
tbClusterService.onEntityStateChange(TenantId.SYS_TENANT_ID, tenantProfile.getId(), |
|||
newTenantProfile ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); |
|||
return tenantProfile; |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfile/{tenantProfileId}", method = RequestMethod.DELETE) |
|||
@ResponseStatus(value = HttpStatus.OK) |
|||
public void deleteTenantProfile(@PathVariable("tenantProfileId") String strTenantProfileId) throws ThingsboardException { |
|||
checkParameter("tenantProfileId", strTenantProfileId); |
|||
try { |
|||
TenantProfileId tenantProfileId = new TenantProfileId(toUUID(strTenantProfileId)); |
|||
TenantProfile profile = checkTenantProfileId(tenantProfileId, Operation.DELETE); |
|||
tenantProfileService.deleteTenantProfile(getTenantId(), tenantProfileId); |
|||
tbClusterService.onTenantProfileDelete(profile, null); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfile/{tenantProfileId}/default", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public TenantProfile setDefaultTenantProfile(@PathVariable("tenantProfileId") String strTenantProfileId) throws ThingsboardException { |
|||
checkParameter("tenantProfileId", strTenantProfileId); |
|||
try { |
|||
TenantProfileId tenantProfileId = new TenantProfileId(toUUID(strTenantProfileId)); |
|||
TenantProfile tenantProfile = checkTenantProfileId(tenantProfileId, Operation.WRITE); |
|||
tenantProfileService.setDefaultTenantProfile(getTenantId(), tenantProfileId); |
|||
return tenantProfile; |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfiles", params = {"pageSize", "page"}, method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public PageData<TenantProfile> getTenantProfiles(@RequestParam int pageSize, |
|||
@RequestParam int page, |
|||
@RequestParam(required = false) String textSearch, |
|||
@RequestParam(required = false) String sortProperty, |
|||
@RequestParam(required = false) String sortOrder) throws ThingsboardException { |
|||
try { |
|||
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); |
|||
return checkNotNull(tenantProfileService.findTenantProfiles(getTenantId(), pageLink)); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
|
|||
@PreAuthorize("hasAuthority('SYS_ADMIN')") |
|||
@RequestMapping(value = "/tenantProfileInfos", params = {"pageSize", "page"}, method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public PageData<EntityInfo> getTenantProfileInfos(@RequestParam int pageSize, |
|||
@RequestParam int page, |
|||
@RequestParam(required = false) String textSearch, |
|||
@RequestParam(required = false) String sortProperty, |
|||
@RequestParam(required = false) String sortOrder) throws ThingsboardException { |
|||
try { |
|||
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); |
|||
return checkNotNull(tenantProfileService.findTenantProfileInfos(getTenantId(), pageLink)); |
|||
} catch (Exception e) { |
|||
throw handleException(e); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,368 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.apiusage; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.checkerframework.checker.nullness.qual.Nullable; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.context.event.ApplicationReadyEvent; |
|||
import org.springframework.context.annotation.Lazy; |
|||
import org.springframework.context.event.EventListener; |
|||
import org.springframework.core.annotation.Order; |
|||
import org.springframework.data.util.Pair; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.ApiFeature; |
|||
import org.thingsboard.server.common.data.ApiUsageRecordKey; |
|||
import org.thingsboard.server.common.data.ApiUsageState; |
|||
import org.thingsboard.server.common.data.ApiUsageStateValue; |
|||
import org.thingsboard.server.common.data.Tenant; |
|||
import org.thingsboard.server.common.data.TenantProfile; |
|||
import org.thingsboard.server.common.data.id.ApiUsageStateId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.TenantProfileId; |
|||
import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
|||
import org.thingsboard.server.common.data.kv.LongDataEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.page.PageDataIterable; |
|||
import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; |
|||
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.tools.SchedulerUtils; |
|||
import org.thingsboard.server.dao.tenant.TenantService; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.scheduler.SchedulerComponent; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.dao.tenant.TbTenantProfileCache; |
|||
import org.thingsboard.server.service.queue.TbClusterService; |
|||
import org.thingsboard.server.service.telemetry.InternalTelemetryService; |
|||
|
|||
import javax.annotation.PostConstruct; |
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ExecutionException; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.concurrent.locks.Lock; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
|
|||
@Slf4j |
|||
@Service |
|||
public class DefaultTbApiUsageStateService implements TbApiUsageStateService { |
|||
|
|||
public static final String HOURLY = "Hourly"; |
|||
public static final FutureCallback<Integer> VOID_CALLBACK = new FutureCallback<Integer>() { |
|||
@Override |
|||
public void onSuccess(@Nullable Integer result) { |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
} |
|||
}; |
|||
private final TbClusterService clusterService; |
|||
private final PartitionService partitionService; |
|||
private final TenantService tenantService; |
|||
private final TimeseriesService tsService; |
|||
private final ApiUsageStateService apiUsageStateService; |
|||
private final SchedulerComponent scheduler; |
|||
private final TbTenantProfileCache tenantProfileCache; |
|||
|
|||
@Lazy |
|||
@Autowired |
|||
private InternalTelemetryService tsWsService; |
|||
|
|||
// Tenants that should be processed on this server
|
|||
private final Map<TenantId, TenantApiUsageState> myTenantStates = new ConcurrentHashMap<>(); |
|||
// Tenants that should be processed on other servers
|
|||
private final Map<TenantId, ApiUsageState> otherTenantStates = new ConcurrentHashMap<>(); |
|||
|
|||
@Value("${usage.stats.report.enabled:true}") |
|||
private boolean enabled; |
|||
|
|||
@Value("${usage.stats.check.cycle:60000}") |
|||
private long nextCycleCheckInterval; |
|||
|
|||
private final Lock updateLock = new ReentrantLock(); |
|||
|
|||
public DefaultTbApiUsageStateService(TbClusterService clusterService, |
|||
PartitionService partitionService, |
|||
TenantService tenantService, |
|||
TimeseriesService tsService, |
|||
ApiUsageStateService apiUsageStateService, |
|||
SchedulerComponent scheduler, |
|||
TbTenantProfileCache tenantProfileCache) { |
|||
this.clusterService = clusterService; |
|||
this.partitionService = partitionService; |
|||
this.tenantService = tenantService; |
|||
this.tsService = tsService; |
|||
this.apiUsageStateService = apiUsageStateService; |
|||
this.scheduler = scheduler; |
|||
this.tenantProfileCache = tenantProfileCache; |
|||
} |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
if (enabled) { |
|||
log.info("Starting api usage service."); |
|||
scheduler.scheduleAtFixedRate(this::checkStartOfNextCycle, nextCycleCheckInterval, nextCycleCheckInterval, TimeUnit.MILLISECONDS); |
|||
log.info("Started api usage service."); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback) { |
|||
ToUsageStatsServiceMsg statsMsg = msg.getValue(); |
|||
TenantId tenantId = new TenantId(new UUID(statsMsg.getTenantIdMSB(), statsMsg.getTenantIdLSB())); |
|||
TenantApiUsageState tenantState; |
|||
List<TsKvEntry> updatedEntries; |
|||
Map<ApiFeature, ApiUsageStateValue> result; |
|||
updateLock.lock(); |
|||
try { |
|||
tenantState = getOrFetchState(tenantId); |
|||
long ts = tenantState.getCurrentCycleTs(); |
|||
long hourTs = tenantState.getCurrentHourTs(); |
|||
long newHourTs = SchedulerUtils.getStartOfCurrentHour(); |
|||
if (newHourTs != hourTs) { |
|||
tenantState.setHour(newHourTs); |
|||
} |
|||
updatedEntries = new ArrayList<>(ApiUsageRecordKey.values().length); |
|||
Set<ApiFeature> apiFeatures = new HashSet<>(); |
|||
for (UsageStatsKVProto kvProto : statsMsg.getValuesList()) { |
|||
ApiUsageRecordKey recordKey = ApiUsageRecordKey.valueOf(kvProto.getKey()); |
|||
long newValue = tenantState.add(recordKey, kvProto.getValue()); |
|||
updatedEntries.add(new BasicTsKvEntry(ts, new LongDataEntry(recordKey.getApiCountKey(), newValue))); |
|||
long newHourlyValue = tenantState.addToHourly(recordKey, kvProto.getValue()); |
|||
updatedEntries.add(new BasicTsKvEntry(hourTs, new LongDataEntry(recordKey.getApiCountKey() + HOURLY, newHourlyValue))); |
|||
apiFeatures.add(recordKey.getApiFeature()); |
|||
} |
|||
result = tenantState.checkStateUpdatedDueToThreshold(apiFeatures); |
|||
} finally { |
|||
updateLock.unlock(); |
|||
} |
|||
tsWsService.saveAndNotifyInternal(tenantId, tenantState.getApiUsageState().getId(), updatedEntries, VOID_CALLBACK); |
|||
if (!result.isEmpty()) { |
|||
persistAndNotify(tenantState, result); |
|||
} |
|||
callback.onSuccess(); |
|||
} |
|||
|
|||
@Override |
|||
public void onApplicationEvent(PartitionChangeEvent partitionChangeEvent) { |
|||
if (partitionChangeEvent.getServiceType().equals(ServiceType.TB_CORE)) { |
|||
myTenantStates.entrySet().removeIf(entry -> !partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); |
|||
otherTenantStates.entrySet().removeIf(entry -> partitionService.resolve(ServiceType.TB_CORE, entry.getKey(), entry.getKey()).isMyPartition()); |
|||
initStatesFromDataBase(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ApiUsageState getApiUsageState(TenantId tenantId) { |
|||
TenantApiUsageState tenantState = myTenantStates.get(tenantId); |
|||
if (tenantState != null) { |
|||
return tenantState.getApiUsageState(); |
|||
} else { |
|||
ApiUsageState state = otherTenantStates.get(tenantId); |
|||
if (state != null) { |
|||
return state; |
|||
} else { |
|||
if (partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { |
|||
return getOrFetchState(tenantId).getApiUsageState(); |
|||
} else { |
|||
updateLock.lock(); |
|||
try { |
|||
state = otherTenantStates.get(tenantId); |
|||
if (state == null) { |
|||
state = apiUsageStateService.findTenantApiUsageState(tenantId); |
|||
if (state != null) { |
|||
otherTenantStates.put(tenantId, state); |
|||
} |
|||
} |
|||
} finally { |
|||
updateLock.unlock(); |
|||
} |
|||
return state; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void onApiUsageStateUpdate(TenantId tenantId) { |
|||
otherTenantStates.remove(tenantId); |
|||
} |
|||
|
|||
@Override |
|||
public void onTenantProfileUpdate(TenantProfileId tenantProfileId) { |
|||
TenantProfile tenantProfile = tenantProfileCache.get(tenantProfileId); |
|||
updateLock.lock(); |
|||
try { |
|||
myTenantStates.values().forEach(state -> { |
|||
if (tenantProfile.getId().equals(state.getTenantProfileId())) { |
|||
updateTenantState(state, tenantProfile); |
|||
} |
|||
}); |
|||
} finally { |
|||
updateLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void onTenantUpdate(TenantId tenantId) { |
|||
TenantProfile tenantProfile = tenantProfileCache.get(tenantId); |
|||
updateLock.lock(); |
|||
try { |
|||
TenantApiUsageState state = myTenantStates.get(tenantId); |
|||
if (state != null && !state.getTenantProfileId().equals(tenantProfile.getId())) { |
|||
updateTenantState(state, tenantProfile); |
|||
} |
|||
} finally { |
|||
updateLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
private void updateTenantState(TenantApiUsageState state, TenantProfile tenantProfile) { |
|||
TenantProfileData oldProfileData = state.getTenantProfileData(); |
|||
state.setTenantProfileId(tenantProfile.getId()); |
|||
state.setTenantProfileData(tenantProfile.getProfileData()); |
|||
Map<ApiFeature, ApiUsageStateValue> result = state.checkStateUpdatedDueToThresholds(); |
|||
if (!result.isEmpty()) { |
|||
persistAndNotify(state, result); |
|||
} |
|||
updateProfileThresholds(state.getTenantId(), state.getApiUsageState().getId(), |
|||
oldProfileData.getConfiguration(), tenantProfile.getProfileData().getConfiguration()); |
|||
} |
|||
|
|||
private void updateProfileThresholds(TenantId tenantId, ApiUsageStateId id, |
|||
TenantProfileConfiguration oldData, TenantProfileConfiguration newData) { |
|||
long ts = System.currentTimeMillis(); |
|||
List<TsKvEntry> profileThresholds = new ArrayList<>(); |
|||
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { |
|||
long newProfileThreshold = newData.getProfileThreshold(key); |
|||
if (oldData == null || oldData.getProfileThreshold(key) != newProfileThreshold) { |
|||
log.info("[{}] Updating profile threshold [{}]:[{}]", tenantId, key, newProfileThreshold); |
|||
profileThresholds.add(new BasicTsKvEntry(ts, new LongDataEntry(key.getApiLimitKey(), newProfileThreshold))); |
|||
} |
|||
} |
|||
if (!profileThresholds.isEmpty()) { |
|||
tsWsService.saveAndNotifyInternal(tenantId, id, profileThresholds, VOID_CALLBACK); |
|||
} |
|||
} |
|||
|
|||
private void persistAndNotify(TenantApiUsageState state, Map<ApiFeature, ApiUsageStateValue> result) { |
|||
log.info("[{}] Detected update of the API state: {}", state.getTenantId(), result); |
|||
apiUsageStateService.update(state.getApiUsageState()); |
|||
clusterService.onApiStateChange(state.getApiUsageState(), null); |
|||
long ts = System.currentTimeMillis(); |
|||
List<TsKvEntry> stateTelemetry = new ArrayList<>(); |
|||
result.forEach(((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name()))))); |
|||
tsWsService.saveAndNotifyInternal(state.getTenantId(), state.getApiUsageState().getId(), stateTelemetry, VOID_CALLBACK); |
|||
//TODO: notify tenant admin via email!
|
|||
} |
|||
|
|||
private void checkStartOfNextCycle() { |
|||
updateLock.lock(); |
|||
try { |
|||
long now = System.currentTimeMillis(); |
|||
myTenantStates.values().forEach(state -> { |
|||
if ((state.getNextCycleTs() > now) && (state.getNextCycleTs() - now < TimeUnit.HOURS.toMillis(1))) { |
|||
state.setCycles(state.getNextCycleTs(), SchedulerUtils.getStartOfNextNextMonth()); |
|||
} |
|||
}); |
|||
} finally { |
|||
updateLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
private TenantApiUsageState getOrFetchState(TenantId tenantId) { |
|||
TenantApiUsageState tenantState = myTenantStates.get(tenantId); |
|||
if (tenantState == null) { |
|||
ApiUsageState dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); |
|||
if (dbStateEntity == null) { |
|||
try { |
|||
dbStateEntity = apiUsageStateService.createDefaultApiUsageState(tenantId); |
|||
} catch (Exception e) { |
|||
dbStateEntity = apiUsageStateService.findTenantApiUsageState(tenantId); |
|||
} |
|||
} |
|||
TenantProfile tenantProfile = tenantProfileCache.get(tenantId); |
|||
tenantState = new TenantApiUsageState(tenantProfile, dbStateEntity); |
|||
try { |
|||
List<TsKvEntry> dbValues = tsService.findAllLatest(tenantId, dbStateEntity.getId()).get(); |
|||
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { |
|||
boolean cycleEntryFound = false; |
|||
boolean hourlyEntryFound = false; |
|||
for (TsKvEntry tsKvEntry : dbValues) { |
|||
if (tsKvEntry.getKey().equals(key.getApiCountKey())) { |
|||
cycleEntryFound = true; |
|||
tenantState.put(key, tsKvEntry.getTs() == tenantState.getCurrentCycleTs() ? tsKvEntry.getLongValue().get() : 0L); |
|||
} else if (tsKvEntry.getKey().equals(key.getApiCountKey() + HOURLY)) { |
|||
hourlyEntryFound = true; |
|||
tenantState.putHourly(key, tsKvEntry.getTs() == tenantState.getCurrentHourTs() ? tsKvEntry.getLongValue().get() : 0L); |
|||
} |
|||
if (cycleEntryFound && hourlyEntryFound) { |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
log.debug("[{}] Initialized state: {}", tenantId, dbStateEntity); |
|||
myTenantStates.put(tenantId, tenantState); |
|||
} catch (InterruptedException | ExecutionException e) { |
|||
log.warn("[{}] Failed to fetch api usage state from db.", tenantId, e); |
|||
} |
|||
} |
|||
return tenantState; |
|||
} |
|||
|
|||
private void initStatesFromDataBase() { |
|||
try { |
|||
log.info("Initializing tenant states."); |
|||
PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024); |
|||
for (Tenant tenant : tenantIterator) { |
|||
if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) { |
|||
log.debug("[{}] Initializing tenant state.", tenant.getId()); |
|||
updateLock.lock(); |
|||
try { |
|||
updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId())); |
|||
log.debug("[{}] Initialized tenant state.", tenant.getId()); |
|||
} catch (Exception e) { |
|||
log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e); |
|||
} finally { |
|||
updateLock.unlock(); |
|||
} |
|||
} |
|||
} |
|||
log.info("Initialized tenant states."); |
|||
} catch (Exception e) { |
|||
log.warn("Unknown failure", e); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,38 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.apiusage; |
|||
|
|||
import org.springframework.context.ApplicationListener; |
|||
import org.thingsboard.server.common.data.ApiUsageState; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.TenantProfileId; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.discovery.PartitionChangeEvent; |
|||
|
|||
public interface TbApiUsageStateService extends ApplicationListener<PartitionChangeEvent> { |
|||
|
|||
void process(TbProtoQueueMsg<ToUsageStatsServiceMsg> msg, TbCallback callback); |
|||
|
|||
ApiUsageState getApiUsageState(TenantId tenantId); |
|||
|
|||
void onTenantProfileUpdate(TenantProfileId tenantProfileId); |
|||
|
|||
void onTenantUpdate(TenantId tenantId); |
|||
|
|||
void onApiUsageStateUpdate(TenantId tenantId); |
|||
} |
|||
@ -0,0 +1,186 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.apiusage; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.Setter; |
|||
import org.springframework.data.util.Pair; |
|||
import org.thingsboard.server.common.data.ApiFeature; |
|||
import org.thingsboard.server.common.data.ApiUsageRecordKey; |
|||
import org.thingsboard.server.common.data.ApiUsageState; |
|||
import org.thingsboard.server.common.data.ApiUsageStateValue; |
|||
import org.thingsboard.server.common.data.TenantProfile; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.TenantProfileId; |
|||
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; |
|||
import org.thingsboard.server.common.msg.tools.SchedulerUtils; |
|||
|
|||
import java.util.Arrays; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
|
|||
public class TenantApiUsageState { |
|||
|
|||
private final Map<ApiUsageRecordKey, Long> currentCycleValues = new ConcurrentHashMap<>(); |
|||
private final Map<ApiUsageRecordKey, Long> currentHourValues = new ConcurrentHashMap<>(); |
|||
|
|||
@Getter |
|||
@Setter |
|||
private TenantProfileId tenantProfileId; |
|||
@Getter |
|||
@Setter |
|||
private TenantProfileData tenantProfileData; |
|||
@Getter |
|||
private final ApiUsageState apiUsageState; |
|||
@Getter |
|||
private volatile long currentCycleTs; |
|||
@Getter |
|||
private volatile long nextCycleTs; |
|||
@Getter |
|||
private volatile long currentHourTs; |
|||
|
|||
public TenantApiUsageState(TenantProfile tenantProfile, ApiUsageState apiUsageState) { |
|||
this.tenantProfileId = tenantProfile.getId(); |
|||
this.tenantProfileData = tenantProfile.getProfileData(); |
|||
this.apiUsageState = apiUsageState; |
|||
this.currentCycleTs = SchedulerUtils.getStartOfCurrentMonth(); |
|||
this.nextCycleTs = SchedulerUtils.getStartOfNextMonth(); |
|||
this.currentHourTs = SchedulerUtils.getStartOfCurrentHour(); |
|||
} |
|||
|
|||
public void put(ApiUsageRecordKey key, Long value) { |
|||
currentCycleValues.put(key, value); |
|||
} |
|||
|
|||
public void putHourly(ApiUsageRecordKey key, Long value) { |
|||
currentHourValues.put(key, value); |
|||
} |
|||
|
|||
public long add(ApiUsageRecordKey key, long value) { |
|||
long result = currentCycleValues.getOrDefault(key, 0L) + value; |
|||
currentCycleValues.put(key, result); |
|||
return result; |
|||
} |
|||
|
|||
public long get(ApiUsageRecordKey key) { |
|||
return currentCycleValues.getOrDefault(key, 0L); |
|||
} |
|||
|
|||
public long addToHourly(ApiUsageRecordKey key, long value) { |
|||
long result = currentHourValues.getOrDefault(key, 0L) + value; |
|||
currentHourValues.put(key, result); |
|||
return result; |
|||
} |
|||
|
|||
public void setHour(long currentHourTs) { |
|||
this.currentHourTs = currentHourTs; |
|||
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { |
|||
currentHourValues.put(key, 0L); |
|||
} |
|||
} |
|||
|
|||
public void setCycles(long currentCycleTs, long nextCycleTs) { |
|||
this.currentCycleTs = currentCycleTs; |
|||
this.nextCycleTs = nextCycleTs; |
|||
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) { |
|||
currentCycleValues.put(key, 0L); |
|||
} |
|||
} |
|||
|
|||
public long getProfileThreshold(ApiUsageRecordKey key) { |
|||
return tenantProfileData.getConfiguration().getProfileThreshold(key); |
|||
} |
|||
|
|||
public long getProfileWarnThreshold(ApiUsageRecordKey key) { |
|||
return tenantProfileData.getConfiguration().getWarnThreshold(key); |
|||
} |
|||
|
|||
public TenantId getTenantId() { |
|||
return apiUsageState.getTenantId(); |
|||
} |
|||
|
|||
public ApiUsageStateValue getFeatureValue(ApiFeature feature) { |
|||
switch (feature) { |
|||
case TRANSPORT: |
|||
return apiUsageState.getTransportState(); |
|||
case RE: |
|||
return apiUsageState.getReExecState(); |
|||
case DB: |
|||
return apiUsageState.getDbStorageState(); |
|||
case JS: |
|||
return apiUsageState.getJsExecState(); |
|||
default: |
|||
return ApiUsageStateValue.ENABLED; |
|||
} |
|||
} |
|||
|
|||
public boolean setFeatureValue(ApiFeature feature, ApiUsageStateValue value) { |
|||
ApiUsageStateValue currentValue = getFeatureValue(feature); |
|||
switch (feature) { |
|||
case TRANSPORT: |
|||
apiUsageState.setTransportState(value); |
|||
break; |
|||
case RE: |
|||
apiUsageState.setReExecState(value); |
|||
break; |
|||
case DB: |
|||
apiUsageState.setDbStorageState(value); |
|||
break; |
|||
case JS: |
|||
apiUsageState.setJsExecState(value); |
|||
break; |
|||
} |
|||
return !currentValue.equals(value); |
|||
} |
|||
|
|||
public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThresholds() { |
|||
return checkStateUpdatedDueToThreshold(new HashSet<>(Arrays.asList(ApiFeature.values()))); |
|||
} |
|||
|
|||
public Map<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(Set<ApiFeature> features) { |
|||
Map<ApiFeature, ApiUsageStateValue> result = new HashMap<>(); |
|||
for (ApiFeature feature : features) { |
|||
Pair<ApiFeature, ApiUsageStateValue> tmp = checkStateUpdatedDueToThreshold(feature); |
|||
if (tmp != null) { |
|||
result.put(tmp.getFirst(), tmp.getSecond()); |
|||
} |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
public Pair<ApiFeature, ApiUsageStateValue> checkStateUpdatedDueToThreshold(ApiFeature feature) { |
|||
ApiUsageStateValue featureValue = ApiUsageStateValue.ENABLED; |
|||
for (ApiUsageRecordKey recordKey : ApiUsageRecordKey.getKeys(feature)) { |
|||
long value = get(recordKey); |
|||
long threshold = getProfileThreshold(recordKey); |
|||
long warnThreshold = getProfileWarnThreshold(recordKey); |
|||
ApiUsageStateValue tmpValue; |
|||
if (threshold == 0 || value < warnThreshold) { |
|||
tmpValue = ApiUsageStateValue.ENABLED; |
|||
} else if (value < threshold) { |
|||
tmpValue = ApiUsageStateValue.WARNING; |
|||
} else { |
|||
tmpValue = ApiUsageStateValue.DISABLED; |
|||
} |
|||
featureValue = ApiUsageStateValue.toMoreRestricted(featureValue, tmpValue); |
|||
} |
|||
return setFeatureValue(feature, featureValue) ? Pair.of(feature, featureValue) : null; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,257 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.device; |
|||
|
|||
import com.fasterxml.jackson.core.JsonProcessingException; |
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.fasterxml.jackson.databind.node.ObjectNode; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.util.StringUtils; |
|||
import org.thingsboard.server.common.data.DataConstants; |
|||
import org.thingsboard.server.common.data.Device; |
|||
import org.thingsboard.server.common.data.DeviceProfile; |
|||
import org.thingsboard.server.common.data.audit.ActionType; |
|||
import org.thingsboard.server.common.data.id.CustomerId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.UserId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.security.DeviceCredentials; |
|||
import org.thingsboard.server.common.msg.TbMsg; |
|||
import org.thingsboard.server.common.msg.TbMsgMetaData; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.audit.AuditLogService; |
|||
import org.thingsboard.server.dao.device.DeviceCredentialsService; |
|||
import org.thingsboard.server.dao.device.DeviceDao; |
|||
import org.thingsboard.server.dao.device.DeviceProfileDao; |
|||
import org.thingsboard.server.dao.device.DeviceProvisionService; |
|||
import org.thingsboard.server.dao.device.DeviceService; |
|||
import org.thingsboard.server.dao.device.provision.ProvisionFailedException; |
|||
import org.thingsboard.server.dao.device.provision.ProvisionRequest; |
|||
import org.thingsboard.server.dao.device.provision.ProvisionResponse; |
|||
import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; |
|||
import org.thingsboard.server.dao.util.mapping.JacksonUtil; |
|||
import org.thingsboard.server.gen.transport.TransportProtos; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueProducer; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.provider.TbQueueProducerProvider; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.state.DeviceStateService; |
|||
|
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
import java.util.concurrent.ExecutionException; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
|
|||
|
|||
@Service |
|||
@Slf4j |
|||
@TbCoreComponent |
|||
public class DeviceProvisionServiceImpl implements DeviceProvisionService { |
|||
|
|||
protected TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> ruleEngineMsgProducer; |
|||
|
|||
private static final String DEVICE_PROVISION_STATE = "provisionState"; |
|||
private static final String PROVISIONED_STATE = "provisioned"; |
|||
|
|||
private final ReentrantLock deviceCreationLock = new ReentrantLock(); |
|||
|
|||
@Autowired |
|||
DeviceDao deviceDao; |
|||
|
|||
@Autowired |
|||
DeviceProfileDao deviceProfileDao; |
|||
|
|||
@Autowired |
|||
DeviceService deviceService; |
|||
|
|||
@Autowired |
|||
DeviceCredentialsService deviceCredentialsService; |
|||
|
|||
@Autowired |
|||
AttributesService attributesService; |
|||
|
|||
@Autowired |
|||
DeviceStateService deviceStateService; |
|||
|
|||
@Autowired |
|||
AuditLogService auditLogService; |
|||
|
|||
@Autowired |
|||
PartitionService partitionService; |
|||
|
|||
public DeviceProvisionServiceImpl(TbQueueProducerProvider producerProvider) { |
|||
ruleEngineMsgProducer = producerProvider.getRuleEngineMsgProducer(); |
|||
} |
|||
|
|||
@Override |
|||
public ProvisionResponse provisionDevice(ProvisionRequest provisionRequest) { |
|||
String provisionRequestKey = provisionRequest.getCredentials().getProvisionDeviceKey(); |
|||
String provisionRequestSecret = provisionRequest.getCredentials().getProvisionDeviceSecret(); |
|||
|
|||
if (StringUtils.isEmpty(provisionRequestKey) || StringUtils.isEmpty(provisionRequestSecret)) { |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.NOT_FOUND.name()); |
|||
} |
|||
|
|||
DeviceProfile targetProfile = deviceProfileDao.findByProvisionDeviceKey(provisionRequestKey); |
|||
|
|||
if (targetProfile == null || targetProfile.getProfileData().getProvisionConfiguration() == null || |
|||
targetProfile.getProfileData().getProvisionConfiguration().getProvisionDeviceSecret() == null) { |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.NOT_FOUND.name()); |
|||
} |
|||
|
|||
Device targetDevice = deviceDao.findDeviceByTenantIdAndName(targetProfile.getTenantId().getId(), provisionRequest.getDeviceName()).orElse(null); |
|||
|
|||
switch (targetProfile.getProvisionType()) { |
|||
case ALLOW_CREATE_NEW_DEVICES: |
|||
if (targetProfile.getProfileData().getProvisionConfiguration().getProvisionDeviceSecret().equals(provisionRequestSecret)) { |
|||
if (targetDevice != null) { |
|||
log.warn("[{}] The device is present and could not be provisioned once more!", targetDevice.getName()); |
|||
notify(targetDevice, provisionRequest, DataConstants.PROVISION_FAILURE, false); |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); |
|||
} else { |
|||
return createDevice(provisionRequest, targetProfile); |
|||
} |
|||
} |
|||
break; |
|||
case CHECK_PRE_PROVISIONED_DEVICES: |
|||
if (targetProfile.getProfileData().getProvisionConfiguration().getProvisionDeviceSecret().equals(provisionRequestSecret)) { |
|||
if (targetDevice != null && targetDevice.getDeviceProfileId().equals(targetProfile.getId())) { |
|||
return processProvision(targetDevice, provisionRequest); |
|||
} else { |
|||
log.warn("[{}] Failed to find pre provisioned device!", provisionRequest.getDeviceName()); |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); |
|||
} |
|||
} |
|||
break; |
|||
} |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.NOT_FOUND.name()); |
|||
} |
|||
|
|||
private ProvisionResponse processProvision(Device device, ProvisionRequest provisionRequest) { |
|||
try { |
|||
Optional<AttributeKvEntry> provisionState = attributesService.find(device.getTenantId(), device.getId(), |
|||
DataConstants.SERVER_SCOPE, DEVICE_PROVISION_STATE).get(); |
|||
if (provisionState != null && provisionState.isPresent() && !provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { |
|||
notify(device, provisionRequest, DataConstants.PROVISION_FAILURE, false); |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); |
|||
} else { |
|||
saveProvisionStateAttribute(device).get(); |
|||
notify(device, provisionRequest, DataConstants.PROVISION_SUCCESS, true); |
|||
} |
|||
} catch (InterruptedException | ExecutionException e) { |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); |
|||
} |
|||
return new ProvisionResponse(deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId()), ProvisionResponseStatus.SUCCESS); |
|||
} |
|||
|
|||
private ProvisionResponse createDevice(ProvisionRequest provisionRequest, DeviceProfile profile) { |
|||
deviceCreationLock.lock(); |
|||
try { |
|||
return processCreateDevice(provisionRequest, profile); |
|||
} finally { |
|||
deviceCreationLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
private void notify(Device device, ProvisionRequest provisionRequest, String type, boolean success) { |
|||
pushProvisionEventToRuleEngine(provisionRequest, device, type); |
|||
logAction(device.getTenantId(), device.getCustomerId(), device, success, provisionRequest); |
|||
} |
|||
|
|||
private ProvisionResponse processCreateDevice(ProvisionRequest provisionRequest, DeviceProfile profile) { |
|||
Device device = deviceService.findDeviceByTenantIdAndName(profile.getTenantId(), provisionRequest.getDeviceName()); |
|||
try { |
|||
if (device == null) { |
|||
Device savedDevice = deviceService.saveDevice(provisionRequest, profile); |
|||
|
|||
deviceStateService.onDeviceAdded(savedDevice); |
|||
saveProvisionStateAttribute(savedDevice).get(); |
|||
pushDeviceCreatedEventToRuleEngine(savedDevice); |
|||
notify(savedDevice, provisionRequest, DataConstants.PROVISION_SUCCESS, true); |
|||
|
|||
return new ProvisionResponse(getDeviceCredentials(savedDevice), ProvisionResponseStatus.SUCCESS); |
|||
} else { |
|||
log.warn("[{}] The device is already provisioned!", device.getName()); |
|||
notify(device, provisionRequest, DataConstants.PROVISION_FAILURE, false); |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); |
|||
} |
|||
} catch (InterruptedException | ExecutionException e) { |
|||
throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); |
|||
} |
|||
} |
|||
|
|||
private ListenableFuture<List<Void>> saveProvisionStateAttribute(Device device) { |
|||
return attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, |
|||
Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), |
|||
System.currentTimeMillis()))); |
|||
} |
|||
|
|||
private DeviceCredentials getDeviceCredentials(Device device) { |
|||
return deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId()); |
|||
} |
|||
|
|||
private void pushProvisionEventToRuleEngine(ProvisionRequest request, Device device, String type) { |
|||
try { |
|||
JsonNode entityNode = JacksonUtil.valueToTree(request); |
|||
TbMsg msg = TbMsg.newMsg(type, device.getId(), createTbMsgMetaData(device), JacksonUtil.toString(entityNode)); |
|||
sendToRuleEngine(device.getTenantId(), msg, null); |
|||
} catch (IllegalArgumentException e) { |
|||
log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), type, e); |
|||
} |
|||
} |
|||
|
|||
private void pushDeviceCreatedEventToRuleEngine(Device device) { |
|||
try { |
|||
ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(device); |
|||
TbMsg msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, device.getId(), createTbMsgMetaData(device), JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); |
|||
sendToRuleEngine(device.getTenantId(), msg, null); |
|||
} catch (JsonProcessingException | IllegalArgumentException e) { |
|||
log.warn("[{}] Failed to push device action to rule engine: {}", device.getId(), DataConstants.ENTITY_CREATED, e); |
|||
} |
|||
} |
|||
|
|||
protected void sendToRuleEngine(TenantId tenantId, TbMsg tbMsg, TbQueueCallback callback) { |
|||
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, tbMsg.getOriginator()); |
|||
TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder().setTbMsg(TbMsg.toByteString(tbMsg)) |
|||
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) |
|||
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()).build(); |
|||
ruleEngineMsgProducer.send(tpi, new TbProtoQueueMsg<>(tbMsg.getId(), msg), callback); |
|||
} |
|||
|
|||
private TbMsgMetaData createTbMsgMetaData(Device device) { |
|||
TbMsgMetaData metaData = new TbMsgMetaData(); |
|||
metaData.putValue("tenantId", device.getTenantId().toString()); |
|||
return metaData; |
|||
} |
|||
|
|||
private void logAction(TenantId tenantId, CustomerId customerId, Device device, boolean success, ProvisionRequest provisionRequest) { |
|||
ActionType actionType = success ? ActionType.PROVISION_SUCCESS : ActionType.PROVISION_FAILURE; |
|||
auditLogService.logEntityAction(tenantId, customerId, new UserId(UserId.NULL_UUID), device.getName(), device.getId(), device, actionType, null, provisionRequest); |
|||
} |
|||
} |
|||
@ -1,29 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.encoding; |
|||
|
|||
import org.thingsboard.server.common.msg.TbActorMsg; |
|||
|
|||
import java.util.Optional; |
|||
|
|||
public interface DataDecodingEncodingService { |
|||
|
|||
Optional<TbActorMsg> decode(byte[] byteArray); |
|||
|
|||
byte[] encode(TbActorMsg msq); |
|||
|
|||
} |
|||
|
|||
@ -1,48 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.encoding; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.nustaq.serialization.FSTConfiguration; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.msg.TbActorMsg; |
|||
|
|||
import java.util.Optional; |
|||
|
|||
@Slf4j |
|||
@Service |
|||
public class ProtoWithFSTService implements DataDecodingEncodingService { |
|||
|
|||
private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); |
|||
|
|||
@Override |
|||
public Optional<TbActorMsg> decode(byte[] byteArray) { |
|||
try { |
|||
TbActorMsg msg = (TbActorMsg) config.asObject(byteArray); |
|||
return Optional.of(msg); |
|||
|
|||
} catch (IllegalArgumentException e) { |
|||
log.error("Error during deserialization message, [{}]", e.getMessage()); |
|||
return Optional.empty(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public byte[] encode(TbActorMsg msq) { |
|||
return config.asByteArray(msq); |
|||
} |
|||
|
|||
} |
|||
@ -1,314 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import com.datastax.driver.core.KeyspaceMetadata; |
|||
import com.datastax.driver.core.exceptions.InvalidQueryException; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.dashboard.DashboardService; |
|||
import org.thingsboard.server.dao.util.NoSqlDao; |
|||
import org.thingsboard.server.service.install.cql.CassandraDbHelper; |
|||
|
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
|
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ASSET; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ASSIGNED_CUSTOMERS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.CONFIGURATION; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.CUSTOMER_ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.DASHBOARD; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.DEVICE; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.END_TS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_TYPE; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_VIEW; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ENTITY_VIEWS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.KEYS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.NAME; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.SEARCH_TEXT; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.START_TS; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.TENANT_ID; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.TITLE; |
|||
import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; |
|||
|
|||
@Service |
|||
@NoSqlDao |
|||
@Profile("install") |
|||
@Slf4j |
|||
public class CassandraDatabaseUpgradeService extends AbstractCassandraDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { |
|||
|
|||
private static final String SCHEMA_UPDATE_CQL = "schema_update.cql"; |
|||
|
|||
@Autowired |
|||
private DashboardService dashboardService; |
|||
|
|||
@Autowired |
|||
private InstallScripts installScripts; |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
|
|||
switch (fromVersion) { |
|||
case "1.2.3": |
|||
|
|||
log.info("Upgrading Cassandara DataBase from version {} to 1.3.0 ...", fromVersion); |
|||
|
|||
//Dump devices, assets and relations
|
|||
|
|||
cluster.getSession(); |
|||
|
|||
KeyspaceMetadata ks = cluster.getCluster().getMetadata().getKeyspace(cluster.getKeyspaceName()); |
|||
|
|||
log.info("Dumping devices ..."); |
|||
Path devicesDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), DEVICE, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, |
|||
new String[]{"", "", "", "", "", "", "default"}, |
|||
"tb-devices"); |
|||
log.info("Devices dumped."); |
|||
|
|||
log.info("Dumping assets ..."); |
|||
Path assetsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), ASSET, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, |
|||
new String[]{"", "", "", "", "", "", "default"}, |
|||
"tb-assets"); |
|||
log.info("Assets dumped."); |
|||
|
|||
log.info("Dumping relations ..."); |
|||
Path relationsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), "relation", |
|||
new String[]{"from_id", "from_type", "to_id", "to_type", "relation_type", ADDITIONAL_INFO, "relation_type_group"}, |
|||
new String[]{"", "", "", "", "", "", "COMMON"}, |
|||
"tb-relations"); |
|||
log.info("Relations dumped."); |
|||
|
|||
log.info("Updating schema ..."); |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "1.3.0", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
//Restore devices, assets and relations
|
|||
|
|||
log.info("Restoring devices ..."); |
|||
if (devicesDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), DEVICE, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, devicesDump); |
|||
Files.deleteIfExists(devicesDump); |
|||
} |
|||
log.info("Devices restored."); |
|||
|
|||
log.info("Dumping device types ..."); |
|||
Path deviceTypesDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), DEVICE, |
|||
new String[]{TENANT_ID, "type"}, |
|||
new String[]{"", ""}, |
|||
"tb-device-types"); |
|||
if (deviceTypesDump != null) { |
|||
CassandraDbHelper.appendToEndOfLine(deviceTypesDump, "DEVICE"); |
|||
} |
|||
log.info("Device types dumped."); |
|||
log.info("Loading device types ..."); |
|||
if (deviceTypesDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), "entity_subtype", |
|||
new String[]{TENANT_ID, "type", "entity_type"}, deviceTypesDump); |
|||
Files.deleteIfExists(deviceTypesDump); |
|||
} |
|||
log.info("Device types loaded."); |
|||
|
|||
log.info("Restoring assets ..."); |
|||
if (assetsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), ASSET, |
|||
new String[]{"id", TENANT_ID, CUSTOMER_ID, "name", SEARCH_TEXT, ADDITIONAL_INFO, "type"}, assetsDump); |
|||
Files.deleteIfExists(assetsDump); |
|||
} |
|||
log.info("Assets restored."); |
|||
|
|||
log.info("Dumping asset types ..."); |
|||
Path assetTypesDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), ASSET, |
|||
new String[]{TENANT_ID, "type"}, |
|||
new String[]{"", ""}, |
|||
"tb-asset-types"); |
|||
if (assetTypesDump != null) { |
|||
CassandraDbHelper.appendToEndOfLine(assetTypesDump, "ASSET"); |
|||
} |
|||
log.info("Asset types dumped."); |
|||
log.info("Loading asset types ..."); |
|||
if (assetTypesDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), "entity_subtype", |
|||
new String[]{TENANT_ID, "type", "entity_type"}, assetTypesDump); |
|||
Files.deleteIfExists(assetTypesDump); |
|||
} |
|||
log.info("Asset types loaded."); |
|||
|
|||
log.info("Restoring relations ..."); |
|||
if (relationsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), "relation", |
|||
new String[]{"from_id", "from_type", "to_id", "to_type", "relation_type", ADDITIONAL_INFO, "relation_type_group"}, relationsDump); |
|||
Files.deleteIfExists(relationsDump); |
|||
} |
|||
log.info("Relations restored."); |
|||
|
|||
break; |
|||
case "1.3.0": |
|||
break; |
|||
case "1.3.1": |
|||
|
|||
cluster.getSession(); |
|||
|
|||
ks = cluster.getCluster().getMetadata().getKeyspace(cluster.getKeyspaceName()); |
|||
|
|||
log.info("Dumping dashboards ..."); |
|||
Path dashboardsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), DASHBOARD, |
|||
new String[]{ID, TENANT_ID, CUSTOMER_ID, TITLE, SEARCH_TEXT, ASSIGNED_CUSTOMERS, CONFIGURATION}, |
|||
new String[]{"", "", "", "", "", "", ""}, |
|||
"tb-dashboards", true); |
|||
log.info("Dashboards dumped."); |
|||
|
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "1.4.0", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
log.info("Restoring dashboards ..."); |
|||
if (dashboardsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), DASHBOARD, |
|||
new String[]{ID, TENANT_ID, TITLE, SEARCH_TEXT, CONFIGURATION}, dashboardsDump, true); |
|||
DatabaseHelper.upgradeTo40_assignDashboards(dashboardsDump, dashboardService, false); |
|||
Files.deleteIfExists(dashboardsDump); |
|||
} |
|||
log.info("Dashboards restored."); |
|||
break; |
|||
case "1.4.0": |
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.0.0", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
break; |
|||
|
|||
case "2.0.0": |
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.1.1", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
break; |
|||
|
|||
case "2.1.1": |
|||
|
|||
log.info("Upgrading Cassandra DataBase from version {} to 2.1.2 ...", fromVersion); |
|||
|
|||
cluster.getSession(); |
|||
|
|||
ks = cluster.getCluster().getMetadata().getKeyspace(cluster.getKeyspaceName()); |
|||
|
|||
log.info("Dumping entity views ..."); |
|||
Path entityViewsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), ENTITY_VIEWS, |
|||
new String[]{ID, ENTITY_ID, ENTITY_TYPE, TENANT_ID, CUSTOMER_ID, NAME, TYPE, KEYS, START_TS, END_TS, SEARCH_TEXT, ADDITIONAL_INFO}, |
|||
new String[]{"", "", "", "", "", "", "default", "", "0", "0", "", ""}, |
|||
"tb-entity-views"); |
|||
log.info("Entity views dumped."); |
|||
|
|||
log.info("Updating schema ..."); |
|||
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.1.2", SCHEMA_UPDATE_CQL); |
|||
loadCql(schemaUpdateFile); |
|||
log.info("Schema updated."); |
|||
|
|||
log.info("Restoring entity views ..."); |
|||
if (entityViewsDump != null) { |
|||
CassandraDbHelper.loadCf(ks, cluster.getSession(), ENTITY_VIEW, |
|||
new String[]{ID, ENTITY_ID, ENTITY_TYPE, TENANT_ID, CUSTOMER_ID, NAME, TYPE, KEYS, START_TS, END_TS, SEARCH_TEXT, ADDITIONAL_INFO}, entityViewsDump); |
|||
Files.deleteIfExists(entityViewsDump); |
|||
} |
|||
log.info("Entity views restored."); |
|||
|
|||
break; |
|||
case "2.1.3": |
|||
break; |
|||
case "2.3.0": |
|||
break; |
|||
case "2.3.1": |
|||
log.info("Updating schema ..."); |
|||
String updateDeviceTableStmt = "alter table device add label text"; |
|||
try { |
|||
cluster.getSession().execute(updateDeviceTableStmt); |
|||
Thread.sleep(2500); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
case "2.4.1": |
|||
log.info("Updating schema ..."); |
|||
String updateAssetTableStmt = "alter table asset add label text"; |
|||
try { |
|||
log.info("Updating assets ..."); |
|||
cluster.getSession().execute(updateAssetTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Assets updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
case "2.4.2": |
|||
log.info("Updating schema ..."); |
|||
String updateAlarmTableStmt = "alter table alarm add propagate_relation_types text"; |
|||
try { |
|||
log.info("Updating alarms ..."); |
|||
cluster.getSession().execute(updateAlarmTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Alarms updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
case "2.4.3": |
|||
log.info("Updating schema ..."); |
|||
String updateAttributeKvTableStmt = "alter table attributes_kv_cf add json_v text"; |
|||
try { |
|||
log.info("Updating attributes ..."); |
|||
cluster.getSession().execute(updateAttributeKvTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Attributes updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
|
|||
String updateTenantCoreTableStmt = "alter table tenant add isolated_tb_core boolean"; |
|||
String updateTenantRuleEngineTableStmt = "alter table tenant add isolated_tb_rule_engine boolean"; |
|||
|
|||
try { |
|||
log.info("Updating tenant..."); |
|||
cluster.getSession().execute(updateTenantCoreTableStmt); |
|||
Thread.sleep(2500); |
|||
|
|||
cluster.getSession().execute(updateTenantRuleEngineTableStmt); |
|||
Thread.sleep(2500); |
|||
log.info("Tenant updated."); |
|||
} catch (InvalidQueryException e) { |
|||
} |
|||
log.info("Schema updated."); |
|||
break; |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -1,30 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.NoSqlDao; |
|||
|
|||
@Service |
|||
@NoSqlDao |
|||
@Profile("install") |
|||
public class CassandraEntityDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService |
|||
implements EntityDatabaseSchemaService { |
|||
public CassandraEntityDatabaseSchemaService() { |
|||
super("schema-entities.cql"); |
|||
} |
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.NoSqlTsLatestDao; |
|||
|
|||
@Service |
|||
@NoSqlTsLatestDao |
|||
@Profile("install") |
|||
public class CassandraTsLatestDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService |
|||
implements TsLatestDatabaseSchemaService { |
|||
public CassandraTsLatestDatabaseSchemaService() { |
|||
super("schema-ts-latest.cql"); |
|||
} |
|||
} |
|||
@ -0,0 +1,19 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
public interface TsLatestDatabaseSchemaService extends DatabaseSchemaService { |
|||
} |
|||
@ -0,0 +1,327 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
import org.thingsboard.server.dao.cassandra.CassandraCluster; |
|||
import org.thingsboard.server.dao.util.NoSqlAnyDao; |
|||
import org.thingsboard.server.service.install.EntityDatabaseSchemaService; |
|||
|
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
import java.util.Arrays; |
|||
import java.util.List; |
|||
|
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.bigintColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.booleanColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.doubleColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.enumToIntColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.idColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.jsonColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.stringColumn; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@NoSqlAnyDao |
|||
@Slf4j |
|||
public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateService { |
|||
|
|||
@Autowired |
|||
private EntityDatabaseSchemaService entityDatabaseSchemaService; |
|||
|
|||
@Autowired |
|||
protected CassandraCluster cluster; |
|||
|
|||
@Value("${spring.datasource.url}") |
|||
protected String dbUrl; |
|||
|
|||
@Value("${spring.datasource.username}") |
|||
protected String dbUserName; |
|||
|
|||
@Value("${spring.datasource.password}") |
|||
protected String dbPassword; |
|||
|
|||
@Override |
|||
public void migrate() throws Exception { |
|||
log.info("Performing migration of entities data from cassandra to SQL database ..."); |
|||
entityDatabaseSchemaService.createDatabaseSchema(false); |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
conn.setAutoCommit(false); |
|||
for (CassandraToSqlTable table: tables) { |
|||
table.migrateToSql(cluster.getSession(), conn); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Unexpected error during ThingsBoard entities data migration!", e); |
|||
throw e; |
|||
} |
|||
entityDatabaseSchemaService.createDatabaseIndexes(); |
|||
} |
|||
|
|||
private static List<CassandraToSqlTable> tables = Arrays.asList( |
|||
new CassandraToSqlTable("admin_settings", |
|||
idColumn("id"), |
|||
stringColumn("key"), |
|||
stringColumn("json_value")), |
|||
new CassandraToSqlTable("alarm", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("type"), |
|||
idColumn("originator_id"), |
|||
enumToIntColumn("originator_type", EntityType.class), |
|||
stringColumn("severity"), |
|||
stringColumn("status"), |
|||
bigintColumn("start_ts"), |
|||
bigintColumn("end_ts"), |
|||
bigintColumn("ack_ts"), |
|||
bigintColumn("clear_ts"), |
|||
stringColumn("details", "additional_info"), |
|||
booleanColumn("propagate"), |
|||
stringColumn("propagate_relation_types")), |
|||
new CassandraToSqlTable("asset", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
stringColumn("name"), |
|||
stringColumn("type"), |
|||
stringColumn("label"), |
|||
stringColumn("search_text"), |
|||
stringColumn("additional_info")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("asset_name_unq_key")) { |
|||
this.handleUniqueNameViolation(data, "asset"); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("audit_log_by_tenant_id", "audit_log", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("entity_name"), |
|||
idColumn("user_id"), |
|||
stringColumn("user_name"), |
|||
stringColumn("action_type"), |
|||
stringColumn("action_data"), |
|||
stringColumn("action_status"), |
|||
stringColumn("action_failure_details")), |
|||
new CassandraToSqlTable("attributes_kv_cf", "attribute_kv", |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("attribute_type"), |
|||
stringColumn("attribute_key"), |
|||
booleanColumn("bool_v", true), |
|||
stringColumn("str_v"), |
|||
bigintColumn("long_v"), |
|||
doubleColumn("dbl_v"), |
|||
jsonColumn("json_v"), |
|||
bigintColumn("last_update_ts")), |
|||
new CassandraToSqlTable("component_descriptor", |
|||
idColumn("id"), |
|||
stringColumn("type"), |
|||
stringColumn("scope"), |
|||
stringColumn("name"), |
|||
stringColumn("search_text"), |
|||
stringColumn("clazz"), |
|||
stringColumn("configuration_descriptor"), |
|||
stringColumn("actions")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("component_descriptor_clazz_key")) { |
|||
String clazz = this.getColumnData(data, "clazz").getValue(); |
|||
log.warn("Found component_descriptor record with duplicate clazz [{}]. Record will be ignored!", clazz); |
|||
this.ignoreRecord(batchData, data); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("customer", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text"), |
|||
stringColumn("country"), |
|||
stringColumn("state"), |
|||
stringColumn("city"), |
|||
stringColumn("address"), |
|||
stringColumn("address2"), |
|||
stringColumn("zip"), |
|||
stringColumn("phone"), |
|||
stringColumn("email"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("dashboard", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text"), |
|||
stringColumn("assigned_customers"), |
|||
stringColumn("configuration")), |
|||
new CassandraToSqlTable("device", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
stringColumn("name"), |
|||
stringColumn("type"), |
|||
stringColumn("label"), |
|||
stringColumn("search_text"), |
|||
stringColumn("additional_info")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("device_name_unq_key")) { |
|||
this.handleUniqueNameViolation(data, "device"); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("device_credentials", |
|||
idColumn("id"), |
|||
idColumn("device_id"), |
|||
stringColumn("credentials_type"), |
|||
stringColumn("credentials_id"), |
|||
stringColumn("credentials_value")), |
|||
new CassandraToSqlTable("event", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("event_type"), |
|||
stringColumn("event_uid"), |
|||
stringColumn("body"), |
|||
new CassandraToSqlEventTsColumn()), |
|||
new CassandraToSqlTable("relation", |
|||
idColumn("from_id"), |
|||
stringColumn("from_type"), |
|||
idColumn("to_id"), |
|||
stringColumn("to_type"), |
|||
stringColumn("relation_type_group"), |
|||
stringColumn("relation_type"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("user", "tb_user", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
stringColumn("email"), |
|||
stringColumn("search_text"), |
|||
stringColumn("authority"), |
|||
stringColumn("first_name"), |
|||
stringColumn("last_name"), |
|||
stringColumn("additional_info")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("tb_user_email_key")) { |
|||
this.handleUniqueEmailViolation(data); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("tenant", |
|||
idColumn("id"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text"), |
|||
stringColumn("region"), |
|||
stringColumn("country"), |
|||
stringColumn("state"), |
|||
stringColumn("city"), |
|||
stringColumn("address"), |
|||
stringColumn("address2"), |
|||
stringColumn("zip"), |
|||
stringColumn("phone"), |
|||
stringColumn("email"), |
|||
stringColumn("additional_info"), |
|||
booleanColumn("isolated_tb_core"), |
|||
booleanColumn("isolated_tb_rule_engine")), |
|||
new CassandraToSqlTable("user_credentials", |
|||
idColumn("id"), |
|||
idColumn("user_id"), |
|||
booleanColumn("enabled"), |
|||
stringColumn("password"), |
|||
stringColumn("activate_token"), |
|||
stringColumn("reset_token")) { |
|||
@Override |
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
if (constraint.equalsIgnoreCase("user_credentials_user_id_key")) { |
|||
String id = UUIDConverter.fromString(this.getColumnData(data, "id").getValue()).toString(); |
|||
log.warn("Found user credentials record with duplicate user_id [id:[{}]]. Record will be ignored!", id); |
|||
this.ignoreRecord(batchData, data); |
|||
return true; |
|||
} |
|||
return super.onConstraintViolation(batchData, data, constraint); |
|||
} |
|||
}, |
|||
new CassandraToSqlTable("widget_type", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("bundle_alias"), |
|||
stringColumn("alias"), |
|||
stringColumn("name"), |
|||
stringColumn("descriptor")), |
|||
new CassandraToSqlTable("widgets_bundle", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("alias"), |
|||
stringColumn("title"), |
|||
stringColumn("search_text")), |
|||
new CassandraToSqlTable("rule_chain", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
stringColumn("name"), |
|||
stringColumn("search_text"), |
|||
idColumn("first_rule_node_id"), |
|||
booleanColumn("root"), |
|||
booleanColumn("debug_mode"), |
|||
stringColumn("configuration"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("rule_node", |
|||
idColumn("id"), |
|||
idColumn("rule_chain_id"), |
|||
stringColumn("type"), |
|||
stringColumn("name"), |
|||
booleanColumn("debug_mode"), |
|||
stringColumn("search_text"), |
|||
stringColumn("configuration"), |
|||
stringColumn("additional_info")), |
|||
new CassandraToSqlTable("entity_view", |
|||
idColumn("id"), |
|||
idColumn("tenant_id"), |
|||
idColumn("customer_id"), |
|||
idColumn("entity_id"), |
|||
stringColumn("entity_type"), |
|||
stringColumn("name"), |
|||
stringColumn("type"), |
|||
stringColumn("keys"), |
|||
bigintColumn("start_ts"), |
|||
bigintColumn("end_ts"), |
|||
stringColumn("search_text"), |
|||
stringColumn("additional_info")) |
|||
); |
|||
} |
|||
@ -0,0 +1,178 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import com.datastax.oss.driver.api.core.cql.Row; |
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
|
|||
import java.sql.PreparedStatement; |
|||
import java.sql.SQLException; |
|||
import java.sql.Types; |
|||
import java.util.regex.Pattern; |
|||
|
|||
@Data |
|||
public class CassandraToSqlColumn { |
|||
|
|||
private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); |
|||
private static final String EMPTY_STR = ""; |
|||
|
|||
private int index; |
|||
private int sqlIndex; |
|||
private String cassandraColumnName; |
|||
private String sqlColumnName; |
|||
private CassandraToSqlColumnType type; |
|||
private int sqlType; |
|||
private int size; |
|||
private Class<? extends Enum> enumClass; |
|||
private boolean allowNullBoolean = false; |
|||
|
|||
public static CassandraToSqlColumn idColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.ID); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn stringColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.STRING); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn stringColumn(String cassandraColumnName, String sqlColumnName) { |
|||
return new CassandraToSqlColumn(cassandraColumnName, sqlColumnName); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn bigintColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.BIGINT); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn doubleColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.DOUBLE); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn booleanColumn(String name) { |
|||
return booleanColumn(name, false); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn booleanColumn(String name, boolean allowNullBoolean) { |
|||
return new CassandraToSqlColumn(name, name, CassandraToSqlColumnType.BOOLEAN, null, allowNullBoolean); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn jsonColumn(String name) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.JSON); |
|||
} |
|||
|
|||
public static CassandraToSqlColumn enumToIntColumn(String name, Class<? extends Enum> enumClass) { |
|||
return new CassandraToSqlColumn(name, CassandraToSqlColumnType.ENUM_TO_INT, enumClass); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String columnName) { |
|||
this(columnName, columnName, CassandraToSqlColumnType.STRING, null, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String columnName, CassandraToSqlColumnType type) { |
|||
this(columnName, columnName, type, null, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String columnName, CassandraToSqlColumnType type, Class<? extends Enum> enumClass) { |
|||
this(columnName, columnName, type, enumClass, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String cassandraColumnName, String sqlColumnName) { |
|||
this(cassandraColumnName, sqlColumnName, CassandraToSqlColumnType.STRING, null, false); |
|||
} |
|||
|
|||
public CassandraToSqlColumn(String cassandraColumnName, String sqlColumnName, CassandraToSqlColumnType type, |
|||
Class<? extends Enum> enumClass, boolean allowNullBoolean) { |
|||
this.cassandraColumnName = cassandraColumnName; |
|||
this.sqlColumnName = sqlColumnName; |
|||
this.type = type; |
|||
this.enumClass = enumClass; |
|||
this.allowNullBoolean = allowNullBoolean; |
|||
} |
|||
|
|||
public String getColumnValue(Row row) { |
|||
if (row.isNull(index)) { |
|||
if (this.type == CassandraToSqlColumnType.BOOLEAN && !this.allowNullBoolean) { |
|||
return Boolean.toString(false); |
|||
} else { |
|||
return null; |
|||
} |
|||
} else { |
|||
switch (this.type) { |
|||
case ID: |
|||
return UUIDConverter.fromTimeUUID(row.getUuid(index)); |
|||
case DOUBLE: |
|||
return Double.toString(row.getDouble(index)); |
|||
case INTEGER: |
|||
return Integer.toString(row.getInt(index)); |
|||
case FLOAT: |
|||
return Float.toString(row.getFloat(index)); |
|||
case BIGINT: |
|||
return Long.toString(row.getLong(index)); |
|||
case BOOLEAN: |
|||
return Boolean.toString(row.getBoolean(index)); |
|||
case STRING: |
|||
case JSON: |
|||
case ENUM_TO_INT: |
|||
default: |
|||
String value = row.getString(index); |
|||
return this.replaceNullChars(value); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void setColumnValue(PreparedStatement sqlInsertStatement, String value) throws SQLException { |
|||
if (value == null) { |
|||
sqlInsertStatement.setNull(this.sqlIndex, this.sqlType); |
|||
} else { |
|||
switch (this.type) { |
|||
case DOUBLE: |
|||
sqlInsertStatement.setDouble(this.sqlIndex, Double.parseDouble(value)); |
|||
break; |
|||
case INTEGER: |
|||
sqlInsertStatement.setInt(this.sqlIndex, Integer.parseInt(value)); |
|||
break; |
|||
case FLOAT: |
|||
sqlInsertStatement.setFloat(this.sqlIndex, Float.parseFloat(value)); |
|||
break; |
|||
case BIGINT: |
|||
sqlInsertStatement.setLong(this.sqlIndex, Long.parseLong(value)); |
|||
break; |
|||
case BOOLEAN: |
|||
sqlInsertStatement.setBoolean(this.sqlIndex, Boolean.parseBoolean(value)); |
|||
break; |
|||
case ENUM_TO_INT: |
|||
Enum enumVal = Enum.valueOf(this.enumClass, value); |
|||
int intValue = enumVal.ordinal(); |
|||
sqlInsertStatement.setInt(this.sqlIndex, intValue); |
|||
break; |
|||
case JSON: |
|||
case STRING: |
|||
case ID: |
|||
default: |
|||
sqlInsertStatement.setString(this.sqlIndex, value); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
private String replaceNullChars(String strValue) { |
|||
if (strValue != null) { |
|||
return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); |
|||
} |
|||
return strValue; |
|||
} |
|||
|
|||
} |
|||
|
|||
@ -0,0 +1,64 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import lombok.Data; |
|||
|
|||
@Data |
|||
public class CassandraToSqlColumnData { |
|||
|
|||
private String value; |
|||
private String originalValue; |
|||
private int constraintCounter = 0; |
|||
|
|||
public CassandraToSqlColumnData(String value) { |
|||
this.value = value; |
|||
this.originalValue = value; |
|||
} |
|||
|
|||
public int nextContraintCounter() { |
|||
return ++constraintCounter; |
|||
} |
|||
|
|||
public String getNextConstraintStringValue(CassandraToSqlColumn column) { |
|||
int counter = this.nextContraintCounter(); |
|||
String newValue = this.originalValue + counter; |
|||
int overflow = newValue.length() - column.getSize(); |
|||
if (overflow > 0) { |
|||
newValue = this.originalValue.substring(0, this.originalValue.length()-overflow) + counter; |
|||
} |
|||
return newValue; |
|||
} |
|||
|
|||
public String getNextConstraintEmailValue(CassandraToSqlColumn column) { |
|||
int counter = this.nextContraintCounter(); |
|||
String[] emailValues = this.originalValue.split("@"); |
|||
String newValue = emailValues[0] + "+" + counter + "@" + emailValues[1]; |
|||
int overflow = newValue.length() - column.getSize(); |
|||
if (overflow > 0) { |
|||
newValue = emailValues[0].substring(0, emailValues[0].length()-overflow) + "+" + counter + "@" + emailValues[1]; |
|||
} |
|||
return newValue; |
|||
} |
|||
|
|||
public String getLogValue() { |
|||
if (this.value != null && this.value.length() > 255) { |
|||
return this.value.substring(0, 255) + "...[truncated " + (this.value.length() - 255) + " symbols]"; |
|||
} |
|||
return this.value; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,28 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
public enum CassandraToSqlColumnType { |
|||
ID, |
|||
DOUBLE, |
|||
INTEGER, |
|||
FLOAT, |
|||
BIGINT, |
|||
BOOLEAN, |
|||
STRING, |
|||
JSON, |
|||
ENUM_TO_INT |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import com.datastax.oss.driver.api.core.cql.Row; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; |
|||
|
|||
public class CassandraToSqlEventTsColumn extends CassandraToSqlColumn { |
|||
|
|||
CassandraToSqlEventTsColumn() { |
|||
super("id", "ts", CassandraToSqlColumnType.BIGINT, null, false); |
|||
} |
|||
|
|||
@Override |
|||
public String getColumnValue(Row row) { |
|||
UUID id = row.getUuid(getIndex()); |
|||
long ts = getTs(id); |
|||
return ts + ""; |
|||
} |
|||
|
|||
private long getTs(UUID uuid) { |
|||
return (uuid.timestamp() - EPOCH_DIFF) / 10000; |
|||
} |
|||
} |
|||
@ -0,0 +1,304 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import com.datastax.oss.driver.api.core.cql.ResultSet; |
|||
import com.datastax.oss.driver.api.core.cql.Row; |
|||
import com.datastax.oss.driver.api.core.cql.SimpleStatement; |
|||
import com.datastax.oss.driver.api.core.cql.Statement; |
|||
import lombok.Data; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.hibernate.internal.util.JdbcExceptionHelper; |
|||
import org.postgresql.util.PSQLException; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
import org.thingsboard.server.dao.cassandra.guava.GuavaSession; |
|||
|
|||
import java.sql.Connection; |
|||
import java.sql.DatabaseMetaData; |
|||
import java.sql.PreparedStatement; |
|||
import java.sql.SQLException; |
|||
import java.util.ArrayList; |
|||
import java.util.Arrays; |
|||
import java.util.Iterator; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
|
|||
@Data |
|||
@Slf4j |
|||
public class CassandraToSqlTable { |
|||
|
|||
private static final int DEFAULT_BATCH_SIZE = 10000; |
|||
|
|||
private String cassandraCf; |
|||
private String sqlTableName; |
|||
|
|||
private List<CassandraToSqlColumn> columns; |
|||
|
|||
private int batchSize = DEFAULT_BATCH_SIZE; |
|||
|
|||
private PreparedStatement sqlInsertStatement; |
|||
|
|||
public CassandraToSqlTable(String tableName, CassandraToSqlColumn... columns) { |
|||
this(tableName, tableName, DEFAULT_BATCH_SIZE, columns); |
|||
} |
|||
|
|||
public CassandraToSqlTable(String tableName, String sqlTableName, CassandraToSqlColumn... columns) { |
|||
this(tableName, sqlTableName, DEFAULT_BATCH_SIZE, columns); |
|||
} |
|||
|
|||
public CassandraToSqlTable(String tableName, int batchSize, CassandraToSqlColumn... columns) { |
|||
this(tableName, tableName, batchSize, columns); |
|||
} |
|||
|
|||
public CassandraToSqlTable(String cassandraCf, String sqlTableName, int batchSize, CassandraToSqlColumn... columns) { |
|||
this.cassandraCf = cassandraCf; |
|||
this.sqlTableName = sqlTableName; |
|||
this.batchSize = batchSize; |
|||
this.columns = Arrays.asList(columns); |
|||
for (int i=0;i<columns.length;i++) { |
|||
this.columns.get(i).setIndex(i); |
|||
this.columns.get(i).setSqlIndex(i+1); |
|||
} |
|||
} |
|||
|
|||
public void migrateToSql(GuavaSession session, Connection conn) throws SQLException { |
|||
log.info("[{}] Migrating data from cassandra '{}' Column Family to '{}' SQL table...", this.sqlTableName, this.cassandraCf, this.sqlTableName); |
|||
DatabaseMetaData metadata = conn.getMetaData(); |
|||
java.sql.ResultSet resultSet = metadata.getColumns(null, null, this.sqlTableName, null); |
|||
while (resultSet.next()) { |
|||
String name = resultSet.getString("COLUMN_NAME"); |
|||
int sqlType = resultSet.getInt("DATA_TYPE"); |
|||
int size = resultSet.getInt("COLUMN_SIZE"); |
|||
CassandraToSqlColumn column = this.getColumn(name); |
|||
column.setSize(size); |
|||
column.setSqlType(sqlType); |
|||
} |
|||
this.sqlInsertStatement = createSqlInsertStatement(conn); |
|||
Statement cassandraSelectStatement = createCassandraSelectStatement(); |
|||
cassandraSelectStatement.setPageSize(100); |
|||
ResultSet rs = session.execute(cassandraSelectStatement); |
|||
Iterator<Row> iter = rs.iterator(); |
|||
int rowCounter = 0; |
|||
List<CassandraToSqlColumnData[]> batchData; |
|||
boolean hasNext; |
|||
do { |
|||
batchData = this.extractBatchData(iter); |
|||
hasNext = batchData.size() == this.batchSize; |
|||
this.batchInsert(batchData, conn); |
|||
rowCounter += batchData.size(); |
|||
log.info("[{}] {} records migrated so far...", this.sqlTableName, rowCounter); |
|||
} while (hasNext); |
|||
this.sqlInsertStatement.close(); |
|||
log.info("[{}] {} total records migrated.", this.sqlTableName, rowCounter); |
|||
log.info("[{}] Finished migration data from cassandra '{}' Column Family to '{}' SQL table.", |
|||
this.sqlTableName, this.cassandraCf, this.sqlTableName); |
|||
} |
|||
|
|||
private List<CassandraToSqlColumnData[]> extractBatchData(Iterator<Row> iter) { |
|||
List<CassandraToSqlColumnData[]> batchData = new ArrayList<>(); |
|||
while (iter.hasNext() && batchData.size() < this.batchSize) { |
|||
Row row = iter.next(); |
|||
if (row != null) { |
|||
CassandraToSqlColumnData[] data = this.extractRowData(row); |
|||
batchData.add(data); |
|||
} |
|||
} |
|||
return batchData; |
|||
} |
|||
|
|||
private CassandraToSqlColumnData[] extractRowData(Row row) { |
|||
CassandraToSqlColumnData[] data = new CassandraToSqlColumnData[this.columns.size()]; |
|||
for (CassandraToSqlColumn column: this.columns) { |
|||
String value = column.getColumnValue(row); |
|||
data[column.getIndex()] = new CassandraToSqlColumnData(value); |
|||
} |
|||
return this.validateColumnData(data); |
|||
} |
|||
|
|||
protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { |
|||
for (int i=0;i<data.length;i++) { |
|||
CassandraToSqlColumn column = this.columns.get(i); |
|||
if (column.getType() == CassandraToSqlColumnType.STRING) { |
|||
CassandraToSqlColumnData columnData = data[i]; |
|||
String value = columnData.getValue(); |
|||
if (value != null && value.length() > column.getSize()) { |
|||
log.warn("[{}] Value size [{}] exceeds maximum size [{}] of column [{}] and will be truncated!", |
|||
this.sqlTableName, |
|||
value.length(), column.getSize(), column.getSqlColumnName()); |
|||
log.warn("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
value = value.substring(0, column.getSize()); |
|||
columnData.setOriginalValue(value); |
|||
columnData.setValue(value); |
|||
} |
|||
} |
|||
} |
|||
return data; |
|||
} |
|||
|
|||
protected void batchInsert(List<CassandraToSqlColumnData[]> batchData, Connection conn) throws SQLException { |
|||
boolean retry = false; |
|||
for (CassandraToSqlColumnData[] data : batchData) { |
|||
for (CassandraToSqlColumn column: this.columns) { |
|||
column.setColumnValue(this.sqlInsertStatement, data[column.getIndex()].getValue()); |
|||
} |
|||
try { |
|||
this.sqlInsertStatement.executeUpdate(); |
|||
} catch (SQLException e) { |
|||
if (this.handleInsertException(batchData, data, conn, e)) { |
|||
retry = true; |
|||
break; |
|||
} else { |
|||
throw e; |
|||
} |
|||
} |
|||
} |
|||
if (retry) { |
|||
this.batchInsert(batchData, conn); |
|||
} else { |
|||
conn.commit(); |
|||
} |
|||
} |
|||
|
|||
private boolean handleInsertException(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, |
|||
Connection conn, SQLException ex) throws SQLException { |
|||
conn.commit(); |
|||
String constraint = extractConstraintName(ex).orElse(null); |
|||
if (constraint != null) { |
|||
if (this.onConstraintViolation(batchData, data, constraint)) { |
|||
return true; |
|||
} else { |
|||
log.error("[{}] Unhandled constraint violation [{}] during insert!", this.sqlTableName, constraint); |
|||
log.error("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
} |
|||
} else { |
|||
log.error("[{}] Unhandled exception during insert!", this.sqlTableName); |
|||
log.error("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
private String dataToString(CassandraToSqlColumnData[] data) { |
|||
StringBuffer stringData = new StringBuffer("{\n"); |
|||
for (int i=0;i<data.length;i++) { |
|||
String columnName = this.columns.get(i).getSqlColumnName(); |
|||
String value = data[i].getLogValue(); |
|||
stringData.append("\"").append(columnName).append("\": ").append("[").append(value).append("]\n"); |
|||
} |
|||
stringData.append("}"); |
|||
return stringData.toString(); |
|||
} |
|||
|
|||
protected boolean onConstraintViolation(List<CassandraToSqlColumnData[]> batchData, |
|||
CassandraToSqlColumnData[] data, String constraint) { |
|||
return false; |
|||
} |
|||
|
|||
protected void handleUniqueNameViolation(CassandraToSqlColumnData[] data, String entityType) { |
|||
CassandraToSqlColumn nameColumn = this.getColumn("name"); |
|||
CassandraToSqlColumn searchTextColumn = this.getColumn("search_text"); |
|||
CassandraToSqlColumnData nameColumnData = data[nameColumn.getIndex()]; |
|||
CassandraToSqlColumnData searchTextColumnData = data[searchTextColumn.getIndex()]; |
|||
String prevName = nameColumnData.getValue(); |
|||
String newName = nameColumnData.getNextConstraintStringValue(nameColumn); |
|||
nameColumnData.setValue(newName); |
|||
searchTextColumnData.setValue(searchTextColumnData.getNextConstraintStringValue(searchTextColumn)); |
|||
String id = UUIDConverter.fromString(this.getColumnData(data, "id").getValue()).toString(); |
|||
log.warn("Found {} with duplicate name [id:[{}]]. Attempting to rename {} from '{}' to '{}'...", entityType, id, entityType, prevName, newName); |
|||
} |
|||
|
|||
protected void handleUniqueEmailViolation(CassandraToSqlColumnData[] data) { |
|||
CassandraToSqlColumn emailColumn = this.getColumn("email"); |
|||
CassandraToSqlColumn searchTextColumn = this.getColumn("search_text"); |
|||
CassandraToSqlColumnData emailColumnData = data[emailColumn.getIndex()]; |
|||
CassandraToSqlColumnData searchTextColumnData = data[searchTextColumn.getIndex()]; |
|||
String prevEmail = emailColumnData.getValue(); |
|||
String newEmail = emailColumnData.getNextConstraintEmailValue(emailColumn); |
|||
emailColumnData.setValue(newEmail); |
|||
searchTextColumnData.setValue(searchTextColumnData.getNextConstraintEmailValue(searchTextColumn)); |
|||
String id = UUIDConverter.fromString(this.getColumnData(data, "id").getValue()).toString(); |
|||
log.warn("Found user with duplicate email [id:[{}]]. Attempting to rename email from '{}' to '{}'...", id, prevEmail, newEmail); |
|||
} |
|||
|
|||
protected void ignoreRecord(List<CassandraToSqlColumnData[]> batchData, CassandraToSqlColumnData[] data) { |
|||
log.warn("[{}] Affected data:\n{}", this.sqlTableName, this.dataToString(data)); |
|||
int index = batchData.indexOf(data); |
|||
if (index > 0) { |
|||
batchData.remove(index); |
|||
} |
|||
} |
|||
|
|||
protected CassandraToSqlColumn getColumn(String sqlColumnName) { |
|||
return this.columns.stream().filter(col -> col.getSqlColumnName().equals(sqlColumnName)).findFirst().get(); |
|||
} |
|||
|
|||
protected CassandraToSqlColumnData getColumnData(CassandraToSqlColumnData[] data, String sqlColumnName) { |
|||
CassandraToSqlColumn column = this.getColumn(sqlColumnName); |
|||
return data[column.getIndex()]; |
|||
} |
|||
|
|||
private Optional<String> extractConstraintName(SQLException ex) { |
|||
final String sqlState = JdbcExceptionHelper.extractSqlState( ex ); |
|||
if (sqlState != null) { |
|||
String sqlStateClassCode = JdbcExceptionHelper.determineSqlStateClassCode( sqlState ); |
|||
if ( sqlStateClassCode != null ) { |
|||
if (Arrays.asList( |
|||
"23", // "integrity constraint violation"
|
|||
"27", // "triggered data change violation"
|
|||
"44" // "with check option violation"
|
|||
).contains(sqlStateClassCode)) { |
|||
if (ex instanceof PSQLException) { |
|||
return Optional.of(((PSQLException)ex).getServerErrorMessage().getConstraint()); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
return Optional.empty(); |
|||
} |
|||
|
|||
protected Statement createCassandraSelectStatement() { |
|||
StringBuilder selectStatementBuilder = new StringBuilder(); |
|||
selectStatementBuilder.append("SELECT "); |
|||
for (CassandraToSqlColumn column : columns) { |
|||
selectStatementBuilder.append(column.getCassandraColumnName()).append(","); |
|||
} |
|||
selectStatementBuilder.deleteCharAt(selectStatementBuilder.length() - 1); |
|||
selectStatementBuilder.append(" FROM ").append(cassandraCf); |
|||
return SimpleStatement.newInstance(selectStatementBuilder.toString()); |
|||
} |
|||
|
|||
private PreparedStatement createSqlInsertStatement(Connection conn) throws SQLException { |
|||
StringBuilder insertStatementBuilder = new StringBuilder(); |
|||
insertStatementBuilder.append("INSERT INTO ").append(this.sqlTableName).append(" ("); |
|||
for (CassandraToSqlColumn column : columns) { |
|||
insertStatementBuilder.append(column.getSqlColumnName()).append(","); |
|||
} |
|||
insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); |
|||
insertStatementBuilder.append(") VALUES ("); |
|||
for (CassandraToSqlColumn column : columns) { |
|||
if (column.getType() == CassandraToSqlColumnType.JSON) { |
|||
insertStatementBuilder.append("cast(? AS json)"); |
|||
} else { |
|||
insertStatementBuilder.append("?"); |
|||
} |
|||
insertStatementBuilder.append(","); |
|||
} |
|||
insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); |
|||
insertStatementBuilder.append(")"); |
|||
return conn.prepareStatement(insertStatementBuilder.toString()); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,233 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.hibernate.exception.ConstraintViolationException; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.UUIDConverter; |
|||
import org.thingsboard.server.dao.cassandra.CassandraCluster; |
|||
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; |
|||
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; |
|||
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; |
|||
import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; |
|||
import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; |
|||
import org.thingsboard.server.dao.util.NoSqlTsDao; |
|||
import org.thingsboard.server.dao.util.SqlTsLatestDao; |
|||
import org.thingsboard.server.service.install.InstallScripts; |
|||
|
|||
import java.nio.charset.Charset; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
import java.sql.Connection; |
|||
import java.sql.DriverManager; |
|||
import java.util.Arrays; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.bigintColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.booleanColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.doubleColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.idColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.jsonColumn; |
|||
import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.stringColumn; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@NoSqlTsDao |
|||
@SqlTsLatestDao |
|||
@Slf4j |
|||
public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateService { |
|||
|
|||
private static final int MAX_KEY_LENGTH = 255; |
|||
private static final int MAX_STR_V_LENGTH = 10000000; |
|||
|
|||
@Autowired |
|||
private InsertLatestTsRepository insertLatestTsRepository; |
|||
|
|||
@Autowired |
|||
protected CassandraCluster cluster; |
|||
|
|||
@Autowired |
|||
protected TsKvDictionaryRepository dictionaryRepository; |
|||
|
|||
@Autowired |
|||
private InstallScripts installScripts; |
|||
|
|||
@Value("${spring.datasource.url}") |
|||
protected String dbUrl; |
|||
|
|||
@Value("${spring.datasource.username}") |
|||
protected String dbUserName; |
|||
|
|||
@Value("${spring.datasource.password}") |
|||
protected String dbPassword; |
|||
|
|||
private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>(); |
|||
|
|||
protected static final ReentrantLock tsCreationLock = new ReentrantLock(); |
|||
|
|||
@Override |
|||
public void migrate() throws Exception { |
|||
log.info("Performing migration of latest timeseries data from cassandra to SQL database ..."); |
|||
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.0.1", "schema_ts_latest.sql"); |
|||
loadSql(schemaUpdateFile, conn); |
|||
conn.setAutoCommit(false); |
|||
for (CassandraToSqlTable table : tables) { |
|||
table.migrateToSql(cluster.getSession(), conn); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Unexpected error during ThingsBoard entities data migration!", e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
private List<CassandraToSqlTable> tables = Arrays.asList( |
|||
new CassandraToSqlTable("ts_kv_latest_cf", "ts_kv_latest", |
|||
idColumn("entity_id"), |
|||
stringColumn("key"), |
|||
bigintColumn("ts"), |
|||
booleanColumn("bool_v"), |
|||
stringColumn("str_v"), |
|||
bigintColumn("long_v"), |
|||
doubleColumn("dbl_v"), |
|||
jsonColumn("json_v")) { |
|||
|
|||
@Override |
|||
protected void batchInsert(List<CassandraToSqlColumnData[]> batchData, Connection conn) { |
|||
insertLatestTsRepository |
|||
.saveOrUpdate(batchData.stream().map(data -> getTsKvLatestEntity(data)).collect(Collectors.toList())); |
|||
} |
|||
|
|||
@Override |
|||
protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { |
|||
return data; |
|||
} |
|||
}); |
|||
|
|||
private TsKvLatestEntity getTsKvLatestEntity(CassandraToSqlColumnData[] data) { |
|||
TsKvLatestEntity latestEntity = new TsKvLatestEntity(); |
|||
latestEntity.setEntityId(UUIDConverter.fromString(data[0].getValue())); |
|||
latestEntity.setKey(getOrSaveKeyId(data[1].getValue())); |
|||
latestEntity.setTs(Long.parseLong(data[2].getValue())); |
|||
|
|||
String strV = data[4].getValue(); |
|||
if (strV != null) { |
|||
if (strV.length() > MAX_STR_V_LENGTH) { |
|||
log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [str_v] and will be truncated!", |
|||
strV.length(), MAX_STR_V_LENGTH); |
|||
log.warn("Affected data:\n{}", strV); |
|||
strV = strV.substring(0, MAX_STR_V_LENGTH); |
|||
} |
|||
latestEntity.setStrValue(strV); |
|||
} else { |
|||
Long longV = null; |
|||
try { |
|||
longV = Long.parseLong(data[5].getValue()); |
|||
} catch (Exception e) { |
|||
} |
|||
if (longV != null) { |
|||
latestEntity.setLongValue(longV); |
|||
} else { |
|||
Double doubleV = null; |
|||
try { |
|||
doubleV = Double.parseDouble(data[6].getValue()); |
|||
} catch (Exception e) { |
|||
} |
|||
if (doubleV != null) { |
|||
latestEntity.setDoubleValue(doubleV); |
|||
} else { |
|||
|
|||
String jsonV = data[7].getValue(); |
|||
if (StringUtils.isNoneEmpty(jsonV)) { |
|||
latestEntity.setJsonValue(jsonV); |
|||
} else { |
|||
Boolean boolV = null; |
|||
try { |
|||
boolV = Boolean.parseBoolean(data[3].getValue()); |
|||
} catch (Exception e) { |
|||
} |
|||
if (boolV != null) { |
|||
latestEntity.setBooleanValue(boolV); |
|||
} else { |
|||
log.warn("All values in key-value row are nullable "); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
return latestEntity; |
|||
} |
|||
|
|||
protected Integer getOrSaveKeyId(String strKey) { |
|||
if (strKey.length() > MAX_KEY_LENGTH) { |
|||
log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [key] and will be truncated!", |
|||
strKey.length(), MAX_KEY_LENGTH); |
|||
log.warn("Affected data:\n{}", strKey); |
|||
strKey = strKey.substring(0, MAX_KEY_LENGTH); |
|||
} |
|||
|
|||
Integer keyId = tsKvDictionaryMap.get(strKey); |
|||
if (keyId == null) { |
|||
Optional<TsKvDictionary> tsKvDictionaryOptional; |
|||
tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); |
|||
if (!tsKvDictionaryOptional.isPresent()) { |
|||
tsCreationLock.lock(); |
|||
try { |
|||
tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); |
|||
if (!tsKvDictionaryOptional.isPresent()) { |
|||
TsKvDictionary tsKvDictionary = new TsKvDictionary(); |
|||
tsKvDictionary.setKey(strKey); |
|||
try { |
|||
TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); |
|||
tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); |
|||
keyId = saved.getKeyId(); |
|||
} catch (ConstraintViolationException e) { |
|||
tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); |
|||
TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); |
|||
tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); |
|||
keyId = dictionary.getKeyId(); |
|||
} |
|||
} else { |
|||
keyId = tsKvDictionaryOptional.get().getKeyId(); |
|||
} |
|||
} finally { |
|||
tsCreationLock.unlock(); |
|||
} |
|||
} else { |
|||
keyId = tsKvDictionaryOptional.get().getKeyId(); |
|||
tsKvDictionaryMap.put(strKey, keyId); |
|||
} |
|||
} |
|||
return keyId; |
|||
} |
|||
|
|||
private void loadSql(Path sqlFile, Connection conn) throws Exception { |
|||
String sql = new String(Files.readAllBytes(sqlFile), Charset.forName("UTF-8")); |
|||
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
Thread.sleep(5000); |
|||
} |
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
public interface EntitiesMigrateService { |
|||
|
|||
void migrate() throws Exception; |
|||
|
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
/** |
|||
* Copyright © 2016-2020 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install.migrate; |
|||
|
|||
public interface TsLatestMigrateService { |
|||
|
|||
void migrate() throws Exception; |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue