Igor Kulikov
2 years ago
227 changed files with 3924 additions and 3624 deletions
@ -1,23 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- FIX DASHBOARD TEMPLATES AFTER ANGULAR MIGRATION TO VER.15 |
|||
|
|||
UPDATE dashboard SET configuration = REPLACE(configuration, 'mat-button mat-icon-button', 'mat-icon-button') |
|||
WHERE configuration like '%mat-button mat-icon-button%'; |
|||
|
|||
UPDATE widget_type SET descriptor = REPLACE(descriptor, 'mat-button mat-icon-button', 'mat-icon-button') |
|||
WHERE descriptor like '%mat-button mat-icon-button%'; |
|||
@ -1,196 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
ALTER TABLE notification_rule ADD COLUMN IF NOT EXISTS enabled BOOLEAN NOT NULL DEFAULT true; |
|||
|
|||
-- NOTIFICATION CONFIGS VERSION CONTROL START |
|||
|
|||
ALTER TABLE notification_template |
|||
ADD COLUMN IF NOT EXISTS external_id UUID; |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'uq_notification_template_external_id') THEN |
|||
ALTER TABLE notification_template ADD CONSTRAINT uq_notification_template_external_id UNIQUE (tenant_id, external_id); |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
ALTER TABLE notification_target |
|||
ADD COLUMN IF NOT EXISTS external_id UUID; |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'uq_notification_target_external_id') THEN |
|||
ALTER TABLE notification_target ADD CONSTRAINT uq_notification_target_external_id UNIQUE (tenant_id, external_id); |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
ALTER TABLE notification_rule |
|||
ADD COLUMN IF NOT EXISTS external_id UUID; |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'uq_notification_rule_external_id') THEN |
|||
ALTER TABLE notification_rule ADD CONSTRAINT uq_notification_rule_external_id UNIQUE (tenant_id, external_id); |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- NOTIFICATION CONFIGS VERSION CONTROL END |
|||
|
|||
-- EDGE EVENTS MIGRATION START |
|||
DO |
|||
$$ |
|||
DECLARE table_partition RECORD; |
|||
BEGIN |
|||
-- in case of running the upgrade script a second time: |
|||
IF NOT (SELECT exists(SELECT FROM pg_tables WHERE tablename = 'old_edge_event')) THEN |
|||
ALTER TABLE edge_event RENAME TO old_edge_event; |
|||
CREATE INDEX IF NOT EXISTS idx_old_edge_event_created_time_tmp ON old_edge_event(created_time); |
|||
ALTER INDEX IF EXISTS idx_edge_event_tenant_id_and_created_time RENAME TO idx_old_edge_event_tenant_id_and_created_time; |
|||
|
|||
FOR table_partition IN SELECT tablename AS name, split_part(tablename, '_', 3) AS partition_ts |
|||
FROM pg_tables WHERE tablename LIKE 'edge_event_%' |
|||
LOOP |
|||
EXECUTE format('ALTER TABLE %s RENAME TO old_edge_event_%s', table_partition.name, table_partition.partition_ts); |
|||
END LOOP; |
|||
ELSE |
|||
RAISE NOTICE 'Table old_edge_event already exists, leaving as is'; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
CREATE TABLE IF NOT EXISTS edge_event ( |
|||
seq_id INT GENERATED ALWAYS AS IDENTITY, |
|||
id uuid NOT NULL, |
|||
created_time bigint NOT NULL, |
|||
edge_id uuid, |
|||
edge_event_type varchar(255), |
|||
edge_event_uid varchar(255), |
|||
entity_id uuid, |
|||
edge_event_action varchar(255), |
|||
body varchar(10000000), |
|||
tenant_id uuid, |
|||
ts bigint NOT NULL |
|||
) PARTITION BY RANGE (created_time); |
|||
CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_and_created_time ON edge_event(tenant_id, created_time DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_edge_event_id ON edge_event(id); |
|||
ALTER TABLE IF EXISTS edge_event ALTER COLUMN seq_id SET CYCLE; |
|||
|
|||
CREATE OR REPLACE PROCEDURE migrate_edge_event(IN start_time_ms BIGINT, IN end_time_ms BIGINT, IN partition_size_ms BIGINT) |
|||
LANGUAGE plpgsql AS |
|||
$$ |
|||
DECLARE |
|||
p RECORD; |
|||
partition_end_ts BIGINT; |
|||
BEGIN |
|||
IF (SELECT exists(SELECT FROM pg_tables WHERE tablename = 'old_edge_event')) THEN |
|||
FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_edge_event |
|||
WHERE created_time >= start_time_ms AND created_time < end_time_ms |
|||
LOOP |
|||
partition_end_ts = p.partition_ts + partition_size_ms; |
|||
RAISE NOTICE '[edge_event] Partition to create : [%-%]', p.partition_ts, partition_end_ts; |
|||
EXECUTE format('CREATE TABLE IF NOT EXISTS edge_event_%s PARTITION OF edge_event ' || |
|||
'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts); |
|||
END LOOP; |
|||
INSERT INTO edge_event (id, created_time, edge_id, edge_event_type, edge_event_uid, entity_id, edge_event_action, body, tenant_id, ts) |
|||
SELECT id, created_time, edge_id, edge_event_type, edge_event_uid, entity_id, edge_event_action, body, tenant_id, ts |
|||
FROM old_edge_event |
|||
WHERE created_time >= start_time_ms AND created_time < end_time_ms; |
|||
ELSE |
|||
RAISE NOTICE 'Table old_edge_event does not exists, skipping migration'; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
-- EDGE EVENTS MIGRATION END |
|||
|
|||
ALTER TABLE resource |
|||
ADD COLUMN IF NOT EXISTS etag varchar; |
|||
|
|||
UPDATE resource |
|||
SET etag = encode(sha256(decode(resource.data, 'base64')),'hex') WHERE resource.data is not null; |
|||
|
|||
ALTER TABLE notification_request ALTER COLUMN info SET DATA TYPE varchar(1000000); |
|||
|
|||
DELETE FROM alarm WHERE tenant_id NOT IN (SELECT id FROM tenant); |
|||
|
|||
CREATE TABLE IF NOT EXISTS alarm_types ( |
|||
tenant_id uuid NOT NULL, |
|||
type varchar(255) NOT NULL, |
|||
CONSTRAINT tenant_id_type_unq_key UNIQUE (tenant_id, type), |
|||
CONSTRAINT fk_entity_tenant_id FOREIGN KEY (tenant_id) REFERENCES tenant(id) ON DELETE CASCADE |
|||
); |
|||
|
|||
INSERT INTO alarm_types (tenant_id, type) SELECT DISTINCT tenant_id, type FROM alarm ON CONFLICT (tenant_id, type) DO NOTHING; |
|||
|
|||
ALTER TABLE widgets_bundle ALTER COLUMN description SET DATA TYPE varchar(1024); |
|||
ALTER TABLE widget_type ALTER COLUMN description SET DATA TYPE varchar(1024); |
|||
|
|||
ALTER TABLE widget_type |
|||
ADD COLUMN IF NOT EXISTS fqn varchar(512); |
|||
ALTER TABLE widget_type |
|||
ADD COLUMN IF NOT EXISTS deprecated boolean NOT NULL DEFAULT false; |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'uq_widget_type_fqn') THEN |
|||
UPDATE widget_type SET fqn = concat(widget_type.bundle_alias, '.', widget_type.alias); |
|||
ALTER TABLE widget_type ADD CONSTRAINT uq_widget_type_fqn UNIQUE (tenant_id, fqn); |
|||
ALTER TABLE widget_type DROP COLUMN IF EXISTS alias; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
ALTER TABLE widget_type |
|||
ADD COLUMN IF NOT EXISTS external_id UUID; |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'widget_type_external_id_unq_key') THEN |
|||
ALTER TABLE widget_type ADD CONSTRAINT widget_type_external_id_unq_key UNIQUE (tenant_id, external_id); |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'uq_widgets_bundle_alias') THEN |
|||
ALTER TABLE widgets_bundle ADD CONSTRAINT uq_widgets_bundle_alias UNIQUE (tenant_id, alias); |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
CREATE TABLE IF NOT EXISTS widgets_bundle_widget ( |
|||
widgets_bundle_id uuid NOT NULL, |
|||
widget_type_id uuid NOT NULL, |
|||
widget_type_order int NOT NULL DEFAULT 0, |
|||
CONSTRAINT widgets_bundle_widget_pkey PRIMARY KEY (widgets_bundle_id, widget_type_id), |
|||
CONSTRAINT fk_widgets_bundle FOREIGN KEY (widgets_bundle_id) REFERENCES widgets_bundle(id) ON DELETE CASCADE, |
|||
CONSTRAINT fk_widget_type FOREIGN KEY (widget_type_id) REFERENCES widget_type(id) ON DELETE CASCADE |
|||
); |
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'widget_type' and column_name='bundle_alias') THEN |
|||
INSERT INTO widgets_bundle_widget SELECT wb.id as widgets_bundle_id, wt.id as widget_type_id from widget_type wt left join widgets_bundle wb ON wt.bundle_alias = wb.alias AND wt.tenant_id = wb.tenant_id ON CONFLICT (widgets_bundle_id, widget_type_id) DO NOTHING; |
|||
ALTER TABLE widget_type DROP COLUMN IF EXISTS bundle_alias; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
@ -1,36 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
ALTER TABLE widget_type |
|||
ADD COLUMN IF NOT EXISTS tags text[]; |
|||
|
|||
ALTER TABLE widgets_bundle |
|||
ADD COLUMN IF NOT EXISTS widgets_bundle_order int; |
|||
|
|||
ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS tbel_exec varchar(32); |
|||
UPDATE api_usage_state SET tbel_exec = js_exec WHERE tbel_exec IS NULL; |
|||
|
|||
ALTER TABLE notification DROP CONSTRAINT IF EXISTS fk_notification_request_id; |
|||
ALTER TABLE notification DROP CONSTRAINT IF EXISTS fk_notification_recipient_id; |
|||
CREATE INDEX IF NOT EXISTS idx_notification_notification_request_id ON notification(request_id); |
|||
CREATE INDEX IF NOT EXISTS idx_notification_request_tenant_id ON notification_request(tenant_id); |
|||
|
|||
-- DELETE invalid records from M:N widgets_bundle_widget table caused by the bug in previous upgrade script; |
|||
DELETE |
|||
FROM widgets_bundle_widget wbw |
|||
WHERE (SELECT tenant_id FROM widgets_bundle wb WHERE wb.id = wbw.widgets_bundle_id) != |
|||
(SELECT tenant_id FROM widget_type wt WHERE wt.id = wbw.widget_type_id); |
|||
|
|||
@ -1,26 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
UPDATE rule_node SET |
|||
configuration = (configuration::jsonb || jsonb_build_object( |
|||
'notifyDevice', |
|||
CASE WHEN configuration::jsonb ->> 'notifyDevice' = 'false' THEN false ELSE true END, |
|||
'sendAttributesUpdatedNotification', |
|||
CASE WHEN configuration::jsonb ->> 'sendAttributesUpdatedNotification' = 'true' THEN true ELSE false END, |
|||
'updateAttributesOnlyOnValueChange', |
|||
CASE WHEN configuration::jsonb ->> 'updateAttributesOnlyOnValueChange' = 'false' THEN false ELSE true END)::jsonb)::varchar, |
|||
configuration_version = 2 |
|||
WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode' AND configuration_version = 1; |
|||
@ -1,59 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- RESOURCES UPDATE START |
|||
|
|||
ALTER TABLE resource ADD COLUMN IF NOT EXISTS descriptor varchar; |
|||
ALTER TABLE resource ADD COLUMN IF NOT EXISTS preview bytea; |
|||
ALTER TABLE resource ADD COLUMN IF NOT EXISTS external_id uuid; |
|||
ALTER TABLE resource ADD COLUMN IF NOT EXISTS is_public boolean default true; |
|||
ALTER TABLE resource ADD COLUMN IF NOT EXISTS public_resource_key varchar(32) unique; |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_resource_etag ON resource(tenant_id, etag); |
|||
CREATE INDEX IF NOT EXISTS idx_resource_type_public_resource_key ON resource(resource_type, public_resource_key); |
|||
|
|||
CREATE OR REPLACE FUNCTION generate_resource_public_key() |
|||
RETURNS text AS $$ |
|||
DECLARE |
|||
chars text := 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; |
|||
result text := ''; |
|||
BEGIN |
|||
FOR i IN 1..32 LOOP |
|||
result := result || substr(chars, floor(random()*62)::int + 1, 1); |
|||
END LOOP; |
|||
RETURN result; |
|||
END; |
|||
$$ LANGUAGE plpgsql; |
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'resource' AND column_name = 'data' AND data_type = 'bytea') THEN |
|||
ALTER TABLE resource RENAME COLUMN data TO base64_data; |
|||
ALTER TABLE resource ADD COLUMN data bytea; |
|||
UPDATE resource SET data = decode(base64_data, 'base64') WHERE base64_data IS NOT NULL; |
|||
ALTER TABLE resource DROP COLUMN base64_data; |
|||
ELSE |
|||
UPDATE resource SET public_resource_key = generate_resource_public_key() WHERE resource_type = 'IMAGE' AND public_resource_key IS NULL; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
DROP FUNCTION generate_resource_public_key; |
|||
|
|||
-- RESOURCES UPDATE END |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_edge_id_created_time ON edge_event(tenant_id, edge_id, created_time DESC); |
|||
@ -1,43 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- RULE NODE INDEXES UPDATE START |
|||
|
|||
DROP INDEX IF EXISTS idx_rule_node_type; |
|||
DROP INDEX IF EXISTS idx_rule_node_type_configuration_version; |
|||
CREATE INDEX IF NOT EXISTS idx_rule_node_type_id_configuration_version ON rule_node(type, id, configuration_version); |
|||
|
|||
-- RULE NODE INDEXES UPDATE END |
|||
|
|||
-- RULE NODE QUEUE UPDATE START |
|||
|
|||
ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS queue_name varchar(255); |
|||
ALTER TABLE component_descriptor ADD COLUMN IF NOT EXISTS has_queue_name boolean DEFAULT false; |
|||
|
|||
-- RULE NODE QUEUE UPDATE END |
|||
|
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'user_settings' AND column_name = 'settings' AND data_type = 'jsonb') THEN |
|||
ALTER TABLE user_settings RENAME COLUMN settings to old_settings; |
|||
ALTER TABLE user_settings ADD COLUMN settings jsonb; |
|||
UPDATE user_settings SET settings = old_settings::jsonb WHERE old_settings IS NOT NULL; |
|||
ALTER TABLE user_settings DROP COLUMN old_settings; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
@ -1,27 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- NOTIFICATIONS UPDATE START |
|||
|
|||
ALTER TABLE notification ADD COLUMN IF NOT EXISTS delivery_method VARCHAR(50) NOT NULL default 'WEB'; |
|||
|
|||
DROP INDEX IF EXISTS idx_notification_recipient_id_created_time; |
|||
DROP INDEX IF EXISTS idx_notification_recipient_id_unread; |
|||
|
|||
CREATE INDEX IF NOT EXISTS idx_notification_delivery_method_recipient_id_created_time ON notification(delivery_method, recipient_id, created_time DESC); |
|||
CREATE INDEX IF NOT EXISTS idx_notification_delivery_method_recipient_id_unread ON notification(delivery_method, recipient_id) WHERE status <> 'READ'; |
|||
|
|||
-- NOTIFICATIONS UPDATE END |
|||
@ -1,162 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- UPDATE PUBLIC CUSTOMERS START |
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS ( |
|||
SELECT FROM information_schema.columns |
|||
WHERE table_name = 'customer' AND column_name = 'is_public' |
|||
) THEN |
|||
ALTER TABLE customer ADD COLUMN is_public boolean DEFAULT false; |
|||
UPDATE customer SET is_public = true WHERE title = 'Public'; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- UPDATE PUBLIC CUSTOMERS END |
|||
|
|||
-- create new attribute_kv table schema |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
-- in case of running the upgrade script a second time: |
|||
IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'attribute_kv' and column_name='entity_type') THEN |
|||
DROP VIEW IF EXISTS device_info_view; |
|||
DROP VIEW IF EXISTS device_info_active_attribute_view; |
|||
ALTER INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts RENAME TO idx_attribute_kv_by_key_and_last_update_ts_old; |
|||
IF EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'attribute_kv_pkey') THEN |
|||
ALTER TABLE attribute_kv RENAME CONSTRAINT attribute_kv_pkey TO attribute_kv_pkey_old; |
|||
END IF; |
|||
ALTER TABLE attribute_kv RENAME TO attribute_kv_old; |
|||
CREATE TABLE IF NOT EXISTS attribute_kv |
|||
( |
|||
entity_id uuid, |
|||
attribute_type int, |
|||
attribute_key int, |
|||
bool_v boolean, |
|||
str_v varchar(10000000), |
|||
long_v bigint, |
|||
dbl_v double precision, |
|||
json_v json, |
|||
last_update_ts bigint, |
|||
CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) |
|||
); |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- rename ts_kv_dictionary table to key_dictionary or create table if not exists |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'ts_kv_dictionary') THEN |
|||
ALTER TABLE ts_kv_dictionary RENAME CONSTRAINT ts_key_id_pkey TO key_dictionary_id_pkey; |
|||
ALTER TABLE ts_kv_dictionary RENAME TO key_dictionary; |
|||
ELSE CREATE TABLE IF NOT EXISTS key_dictionary( |
|||
key varchar(255) NOT NULL, |
|||
key_id serial UNIQUE, |
|||
CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) |
|||
); |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- insert keys into key_dictionary |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN |
|||
INSERT INTO key_dictionary(key) SELECT DISTINCT attribute_key FROM attribute_kv_old ON CONFLICT DO NOTHING; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- migrate attributes from attribute_kv_old to attribute_kv |
|||
DO |
|||
$$ |
|||
DECLARE |
|||
row_num_old integer; |
|||
row_num integer; |
|||
BEGIN |
|||
IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN |
|||
INSERT INTO attribute_kv(entity_id, attribute_type, attribute_key, bool_v, str_v, long_v, dbl_v, json_v, last_update_ts) |
|||
SELECT a.entity_id, CASE |
|||
WHEN a.attribute_type = 'CLIENT_SCOPE' THEN 1 |
|||
WHEN a.attribute_type = 'SERVER_SCOPE' THEN 2 |
|||
WHEN a.attribute_type = 'SHARED_SCOPE' THEN 3 |
|||
ELSE 0 |
|||
END, |
|||
k.key_id, a.bool_v, a.str_v, a.long_v, a.dbl_v, a.json_v, a.last_update_ts |
|||
FROM attribute_kv_old a INNER JOIN key_dictionary k ON (a.attribute_key = k.key); |
|||
SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; |
|||
SELECT COUNT(*) INTO row_num FROM attribute_kv; |
|||
RAISE NOTICE 'Migrated % of % rows', row_num, row_num_old; |
|||
DROP TABLE IF EXISTS attribute_kv_old; |
|||
CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); |
|||
END IF; |
|||
EXCEPTION |
|||
WHEN others THEN |
|||
ROLLBACK; |
|||
RAISE EXCEPTION 'Error during COPY: %', SQLERRM; |
|||
END |
|||
$$; |
|||
|
|||
-- OAUTH2 PARAMS ALTER TABLE START |
|||
|
|||
ALTER TABLE oauth2_params |
|||
ADD COLUMN IF NOT EXISTS edge_enabled boolean DEFAULT false; |
|||
|
|||
-- OAUTH2 PARAMS ALTER TABLE END |
|||
|
|||
-- QUEUE STATS UPDATE START |
|||
|
|||
CREATE TABLE IF NOT EXISTS queue_stats ( |
|||
id uuid NOT NULL CONSTRAINT queue_stats_pkey PRIMARY KEY, |
|||
created_time bigint NOT NULL, |
|||
tenant_id uuid NOT NULL, |
|||
queue_name varchar(255) NOT NULL, |
|||
service_id varchar(255) NOT NULL, |
|||
CONSTRAINT queue_stats_name_unq_key UNIQUE (tenant_id, queue_name, service_id) |
|||
); |
|||
|
|||
INSERT INTO queue_stats |
|||
SELECT id, created_time, tenant_id, substring(name FROM 1 FOR position('_' IN name) - 1) AS queue_name, |
|||
substring(name FROM position('_' IN name) + 1) AS service_id |
|||
FROM asset |
|||
WHERE type = 'TbServiceQueue' and name LIKE '%\_%'; |
|||
|
|||
DELETE FROM asset WHERE type='TbServiceQueue'; |
|||
DELETE FROM asset_profile WHERE name ='TbServiceQueue'; |
|||
|
|||
-- QUEUE STATS UPDATE END |
|||
|
|||
-- MOBILE APP SETTINGS TABLE CREATE START |
|||
|
|||
CREATE TABLE IF NOT EXISTS mobile_app_settings ( |
|||
id uuid NOT NULL CONSTRAINT mobile_app_settings_pkey PRIMARY KEY, |
|||
created_time bigint NOT NULL, |
|||
tenant_id uuid NOT NULL, |
|||
use_default_app boolean, |
|||
android_config VARCHAR(1000), |
|||
ios_config VARCHAR(1000), |
|||
qr_code_config VARCHAR(100000), |
|||
CONSTRAINT mobile_app_settings_tenant_id_unq_key UNIQUE (tenant_id) |
|||
); |
|||
|
|||
-- MOBILE APP SETTINGS TABLE CREATE END |
|||
@ -1,215 +0,0 @@ |
|||
-- |
|||
-- Copyright © 2016-2024 The Thingsboard Authors |
|||
-- |
|||
-- Licensed under the Apache License, Version 2.0 (the "License"); |
|||
-- you may not use this file except in compliance with the License. |
|||
-- You may obtain a copy of the License at |
|||
-- |
|||
-- http://www.apache.org/licenses/LICENSE-2.0 |
|||
-- |
|||
-- Unless required by applicable law or agreed to in writing, software |
|||
-- distributed under the License is distributed on an "AS IS" BASIS, |
|||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
-- See the License for the specific language governing permissions and |
|||
-- limitations under the License. |
|||
-- |
|||
|
|||
-- UPDATE RESOURCE SUB TYPE START |
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS ( |
|||
SELECT FROM information_schema.columns |
|||
WHERE table_name = 'resource' AND column_name = 'resource_sub_type' |
|||
) THEN |
|||
ALTER TABLE resource ADD COLUMN resource_sub_type varchar(32); |
|||
UPDATE resource SET resource_sub_type = 'IMAGE' WHERE resource_type = 'IMAGE'; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- UPDATE RESOURCE SUB TYPE END |
|||
|
|||
-- UPDATE WIDGETS BUNDLE START |
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS ( |
|||
SELECT FROM information_schema.columns |
|||
WHERE table_name = 'widgets_bundle' AND column_name = 'scada' |
|||
) THEN |
|||
ALTER TABLE widgets_bundle ADD COLUMN scada boolean NOT NULL DEFAULT false; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- UPDATE WIDGETS BUNDLE END |
|||
|
|||
-- UPDATE WIDGET TYPE START |
|||
|
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF NOT EXISTS ( |
|||
SELECT FROM information_schema.columns |
|||
WHERE table_name = 'widget_type' AND column_name = 'scada' |
|||
) THEN |
|||
ALTER TABLE widget_type ADD COLUMN scada boolean NOT NULL DEFAULT false; |
|||
END IF; |
|||
END; |
|||
$$; |
|||
|
|||
-- UPDATE WIDGET TYPE END |
|||
|
|||
-- KV VERSIONING UPDATE START |
|||
|
|||
CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1; |
|||
CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1; |
|||
|
|||
ALTER TABLE attribute_kv ADD COLUMN IF NOT EXISTS version bigint default 0; |
|||
ALTER TABLE ts_kv_latest ADD COLUMN IF NOT EXISTS version bigint default 0; |
|||
|
|||
-- KV VERSIONING UPDATE END |
|||
|
|||
-- RELATION VERSIONING UPDATE START |
|||
|
|||
CREATE SEQUENCE IF NOT EXISTS relation_version_seq cache 1; |
|||
ALTER TABLE relation ADD COLUMN IF NOT EXISTS version bigint default 0; |
|||
|
|||
-- RELATION VERSIONING UPDATE END |
|||
|
|||
|
|||
-- ENTITIES VERSIONING UPDATE START |
|||
|
|||
ALTER TABLE device ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE device_profile ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE device_credentials ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE asset ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE asset_profile ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE entity_view ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE tb_user ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE customer ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE edge ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE rule_chain ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE dashboard ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE widget_type ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE widgets_bundle ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
ALTER TABLE tenant ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; |
|||
|
|||
-- ENTITIES VERSIONING UPDATE END |
|||
|
|||
-- OAUTH2 UPDATE START |
|||
|
|||
ALTER TABLE IF EXISTS oauth2_mobile RENAME TO mobile_app; |
|||
ALTER TABLE IF EXISTS oauth2_domain RENAME TO domain; |
|||
ALTER TABLE IF EXISTS oauth2_registration RENAME TO oauth2_client; |
|||
|
|||
ALTER TABLE domain ADD COLUMN IF NOT EXISTS oauth2_enabled boolean, |
|||
ADD COLUMN IF NOT EXISTS edge_enabled boolean, |
|||
ADD COLUMN IF NOT EXISTS tenant_id uuid DEFAULT '13814000-1dd2-11b2-8080-808080808080', |
|||
DROP COLUMN IF EXISTS domain_scheme; |
|||
|
|||
-- rename column domain_name to name |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name='domain' and column_name='domain_name') THEN |
|||
ALTER TABLE domain RENAME COLUMN domain_name TO name; |
|||
END IF; |
|||
END |
|||
$$; |
|||
|
|||
-- delete duplicated domains |
|||
DELETE FROM domain d1 USING ( |
|||
SELECT MIN(ctid) as ctid, name |
|||
FROM domain |
|||
GROUP BY name HAVING COUNT(*) > 1 |
|||
) d2 WHERE d1.name = d2.name AND d1.ctid <> d2.ctid; |
|||
|
|||
ALTER TABLE mobile_app ADD COLUMN IF NOT EXISTS oauth2_enabled boolean, |
|||
ADD COLUMN IF NOT EXISTS tenant_id uuid DEFAULT '13814000-1dd2-11b2-8080-808080808080'; |
|||
|
|||
-- delete duplicated apps |
|||
DELETE FROM mobile_app m1 USING ( |
|||
SELECT MIN(ctid) as ctid, pkg_name |
|||
FROM mobile_app |
|||
GROUP BY pkg_name HAVING COUNT(*) > 1 |
|||
) m2 WHERE m1.pkg_name = m2.pkg_name AND m1.ctid <> m2.ctid; |
|||
|
|||
ALTER TABLE oauth2_client ADD COLUMN IF NOT EXISTS tenant_id uuid DEFAULT '13814000-1dd2-11b2-8080-808080808080', |
|||
ADD COLUMN IF NOT EXISTS title varchar(100); |
|||
UPDATE oauth2_client SET title = additional_info::jsonb->>'providerName' WHERE additional_info IS NOT NULL; |
|||
|
|||
CREATE TABLE IF NOT EXISTS domain_oauth2_client ( |
|||
domain_id uuid NOT NULL, |
|||
oauth2_client_id uuid NOT NULL, |
|||
CONSTRAINT fk_domain FOREIGN KEY (domain_id) REFERENCES domain(id) ON DELETE CASCADE, |
|||
CONSTRAINT fk_oauth2_client FOREIGN KEY (oauth2_client_id) REFERENCES oauth2_client(id) ON DELETE CASCADE |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS mobile_app_oauth2_client ( |
|||
mobile_app_id uuid NOT NULL, |
|||
oauth2_client_id uuid NOT NULL, |
|||
CONSTRAINT fk_domain FOREIGN KEY (mobile_app_id) REFERENCES mobile_app(id) ON DELETE CASCADE, |
|||
CONSTRAINT fk_oauth2_client FOREIGN KEY (oauth2_client_id) REFERENCES oauth2_client(id) ON DELETE CASCADE |
|||
); |
|||
|
|||
-- migrate oauth2_params table |
|||
DO |
|||
$$ |
|||
BEGIN |
|||
IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'oauth2_params') THEN |
|||
UPDATE domain SET oauth2_enabled = p.enabled, |
|||
edge_enabled = p.edge_enabled |
|||
FROM oauth2_params p WHERE p.id = domain.oauth2_params_id; |
|||
|
|||
UPDATE mobile_app SET oauth2_enabled = p.enabled |
|||
FROM oauth2_params p WHERE p.id = mobile_app.oauth2_params_id; |
|||
|
|||
INSERT INTO domain_oauth2_client(domain_id, oauth2_client_id) |
|||
(SELECT d.id, r.id FROM domain d LEFT JOIN oauth2_client r on d.oauth2_params_id = r.oauth2_params_id |
|||
WHERE r.platforms IS NULL OR r.platforms IN ('','WEB')); |
|||
|
|||
INSERT INTO mobile_app_oauth2_client(mobile_app_id, oauth2_client_id) |
|||
(SELECT m.id, r.id FROM mobile_app m LEFT JOIN oauth2_client r on m.oauth2_params_id = r.oauth2_params_id |
|||
WHERE r.platforms IS NULL OR r.platforms IN ('','ANDROID','IOS')); |
|||
|
|||
ALTER TABLE mobile_app RENAME CONSTRAINT oauth2_mobile_pkey TO mobile_app_pkey; |
|||
ALTER TABLE domain RENAME CONSTRAINT oauth2_domain_pkey TO domain_pkey; |
|||
ALTER TABLE oauth2_client RENAME CONSTRAINT oauth2_registration_pkey TO oauth2_client_pkey; |
|||
|
|||
ALTER TABLE domain DROP COLUMN oauth2_params_id; |
|||
ALTER TABLE mobile_app DROP COLUMN oauth2_params_id; |
|||
ALTER TABLE oauth2_client DROP COLUMN oauth2_params_id; |
|||
|
|||
ALTER TABLE mobile_app ADD CONSTRAINT mobile_app_unq_key UNIQUE (pkg_name); |
|||
ALTER TABLE domain ADD CONSTRAINT domain_unq_key UNIQUE (name); |
|||
|
|||
DROP TABLE IF EXISTS oauth2_params; |
|||
-- drop deprecated tables |
|||
DROP TABLE IF EXISTS oauth2_client_registration_info; |
|||
DROP TABLE IF EXISTS oauth2_client_registration; |
|||
END IF; |
|||
END |
|||
$$; |
|||
|
|||
-- OAUTH2 UPDATE END |
|||
|
|||
-- USER CREDENTIALS UPDATE START |
|||
|
|||
ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS activate_token_exp_time BIGINT; |
|||
-- Setting 24-hour TTL for existing activation tokens |
|||
UPDATE user_credentials SET activate_token_exp_time = cast(extract(EPOCH FROM NOW()) * 1000 AS BIGINT) + 86400000 |
|||
WHERE activate_token IS NOT NULL AND activate_token_exp_time IS NULL; |
|||
|
|||
ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS reset_token_exp_time BIGINT; |
|||
-- Setting 24-hour TTL for existing password reset tokens |
|||
UPDATE user_credentials SET reset_token_exp_time = cast(extract(EPOCH FROM NOW()) * 1000 AS BIGINT) + 86400000 |
|||
WHERE reset_token IS NOT NULL AND reset_token_exp_time IS NULL; |
|||
|
|||
UPDATE admin_settings SET json_value = (json_value::jsonb || '{"userActivationTokenTtl":24,"passwordResetTokenTtl":24}'::jsonb)::varchar |
|||
WHERE key = 'securitySettings'; |
|||
|
|||
-- USER CREDENTIALS UPDATE END |
|||
@ -0,0 +1,39 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.device; |
|||
|
|||
import lombok.Getter; |
|||
|
|||
@Getter |
|||
public enum TransportSessionCloseReason { |
|||
|
|||
UNKNOWN_REASON(0, "Unknown Reason.", "Session closed with unknown reason."), |
|||
CREDENTIALS_UPDATED(1, "device credentials updated!", "Close session due to device credentials update."), |
|||
MAX_CONCURRENT_SESSIONS_LIMIT_REACHED(2, "max concurrent sessions limit reached per device!", "Remove eldest session (max concurrent sessions limit reached per device)."), |
|||
SESSION_TIMEOUT(3, "session timeout!", "Close session due to session timeout."), |
|||
RPC_DELIVERY_TIMEOUT(4, "RPC delivery failed!", "Close session due to RPC delivery failure."); |
|||
|
|||
private final int protoNumber; |
|||
private final String notificationMessage; |
|||
private final String logMessage; |
|||
|
|||
TransportSessionCloseReason(int protoNumber, String notificationMessage, String logMessage) { |
|||
this.protoNumber = protoNumber; |
|||
this.notificationMessage = notificationMessage; |
|||
this.logMessage = logMessage; |
|||
} |
|||
|
|||
} |
|||
File diff suppressed because it is too large
@ -0,0 +1,55 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edge.rpc; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.edge.EdgeEvent; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.dao.edge.BaseEdgeEventService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.discovery.TopicService; |
|||
import org.thingsboard.server.queue.provider.TbQueueProducerProvider; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
@Slf4j |
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@ConditionalOnExpression("'${queue.type:null}'=='kafka'") |
|||
public class KafkaEdgeEventService extends BaseEdgeEventService { |
|||
|
|||
private final TopicService topicService; |
|||
private final TbQueueProducerProvider producerProvider; |
|||
|
|||
@Override |
|||
public ListenableFuture<Void> saveAsync(EdgeEvent edgeEvent) { |
|||
validateEdgeEvent(edgeEvent); |
|||
|
|||
TopicPartitionInfo tpi = topicService.getEdgeEventNotificationsTopic(edgeEvent.getTenantId(), edgeEvent.getEdgeId()); |
|||
ToEdgeEventNotificationMsg msg = ToEdgeEventNotificationMsg.newBuilder().setEdgeEventMsg(ProtoUtils.toProto(edgeEvent)).build(); |
|||
producerProvider.getTbEdgeEventsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); |
|||
|
|||
return Futures.immediateFuture(null); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,146 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edge.rpc; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import io.grpc.stub.StreamObserver; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.edge.Edge; |
|||
import org.thingsboard.server.common.data.edge.EdgeEvent; |
|||
import org.thingsboard.server.common.data.id.EdgeId; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.gen.edge.v1.DownlinkMsg; |
|||
import org.thingsboard.server.gen.edge.v1.ResponseMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; |
|||
import org.thingsboard.server.queue.TbQueueConsumer; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; |
|||
import org.thingsboard.server.queue.discovery.TopicService; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaAdmin; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaSettings; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; |
|||
import org.thingsboard.server.queue.provider.TbCoreQueueFactory; |
|||
import org.thingsboard.server.service.edge.EdgeContextComponent; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.ScheduledExecutorService; |
|||
import java.util.function.BiConsumer; |
|||
|
|||
@Slf4j |
|||
public class KafkaEdgeGrpcSession extends EdgeGrpcSession { |
|||
|
|||
private final TopicService topicService; |
|||
private final TbCoreQueueFactory tbCoreQueueFactory; |
|||
|
|||
private final TbKafkaSettings kafkaSettings; |
|||
private final TbKafkaTopicConfigs kafkaTopicConfigs; |
|||
|
|||
private volatile boolean isHighPriorityProcessing; |
|||
|
|||
private QueueConsumerManager<TbProtoQueueMsg<ToEdgeEventNotificationMsg>> consumer; |
|||
|
|||
private ExecutorService consumerExecutor; |
|||
|
|||
public KafkaEdgeGrpcSession(EdgeContextComponent ctx, TopicService topicService, TbCoreQueueFactory tbCoreQueueFactory, |
|||
TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs kafkaTopicConfigs, StreamObserver<ResponseMsg> outputStream, |
|||
BiConsumer<EdgeId, EdgeGrpcSession> sessionOpenListener, BiConsumer<Edge, UUID> sessionCloseListener, |
|||
ScheduledExecutorService sendDownlinkExecutorService, int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { |
|||
super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); |
|||
this.topicService = topicService; |
|||
this.tbCoreQueueFactory = tbCoreQueueFactory; |
|||
this.kafkaSettings = kafkaSettings; |
|||
this.kafkaTopicConfigs = kafkaTopicConfigs; |
|||
} |
|||
|
|||
private void processMsgs(List<TbProtoQueueMsg<ToEdgeEventNotificationMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeEventNotificationMsg>> consumer) { |
|||
log.trace("[{}][{}] starting processing edge events", tenantId, sessionId); |
|||
if (isConnected() && isSyncCompleted() && !isHighPriorityProcessing) { |
|||
List<EdgeEvent> edgeEvents = new ArrayList<>(); |
|||
for (TbProtoQueueMsg<ToEdgeEventNotificationMsg> msg : msgs) { |
|||
EdgeEvent edgeEvent = ProtoUtils.fromProto(msg.getValue().getEdgeEventMsg()); |
|||
edgeEvents.add(edgeEvent); |
|||
} |
|||
List<DownlinkMsg> downlinkMsgsPack = convertToDownlinkMsgsPack(edgeEvents); |
|||
try { |
|||
boolean isInterrupted = sendDownlinkMsgsPack(downlinkMsgsPack).get(); |
|||
if (isInterrupted) { |
|||
log.debug("[{}][{}][{}] Send downlink messages task was interrupted", tenantId, edge.getId(), sessionId); |
|||
} else { |
|||
consumer.commit(); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("[{}] Failed to process all downlink messages", sessionId, e); |
|||
} |
|||
} else { |
|||
try { |
|||
Thread.sleep(ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval()); |
|||
} catch (InterruptedException interruptedException) { |
|||
log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); |
|||
} |
|||
log.trace("[{}][{}] edge is not connected or sync is not completed. Skipping iteration", tenantId, sessionId); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Boolean> migrateEdgeEvents() throws Exception { |
|||
return super.processEdgeEvents(); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Boolean> processEdgeEvents() { |
|||
if (consumer == null) { |
|||
this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edge-event-consumer")); |
|||
this.consumer = QueueConsumerManager.<TbProtoQueueMsg<ToEdgeEventNotificationMsg>>builder() |
|||
.name("TB Edge events") |
|||
.msgPackProcessor(this::processMsgs) |
|||
.pollInterval(ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval()) |
|||
.consumerCreator(() -> tbCoreQueueFactory.createEdgeEventMsgConsumer(tenantId, edge.getId())) |
|||
.consumerExecutor(consumerExecutor) |
|||
.threadPrefix("edge-events") |
|||
.build(); |
|||
consumer.subscribe(); |
|||
consumer.launch(); |
|||
} |
|||
return Futures.immediateFuture(Boolean.FALSE); |
|||
} |
|||
|
|||
@Override |
|||
public void processHighPriorityEvents() { |
|||
isHighPriorityProcessing = true; |
|||
super.processHighPriorityEvents(); |
|||
isHighPriorityProcessing = false; |
|||
} |
|||
|
|||
@Override |
|||
public void destroy() { |
|||
consumer.stop(); |
|||
consumerExecutor.shutdown(); |
|||
} |
|||
|
|||
@Override |
|||
public void cleanUp() { |
|||
String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); |
|||
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); |
|||
kafkaAdmin.deleteTopic(topic); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,46 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edge.rpc; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import io.grpc.stub.StreamObserver; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.common.data.edge.Edge; |
|||
import org.thingsboard.server.common.data.id.EdgeId; |
|||
import org.thingsboard.server.gen.edge.v1.ResponseMsg; |
|||
import org.thingsboard.server.service.edge.EdgeContextComponent; |
|||
|
|||
import java.util.UUID; |
|||
import java.util.concurrent.ScheduledExecutorService; |
|||
import java.util.function.BiConsumer; |
|||
|
|||
@Slf4j |
|||
public class PostgresEdgeGrpcSession extends EdgeGrpcSession { |
|||
|
|||
PostgresEdgeGrpcSession(EdgeContextComponent ctx, StreamObserver<ResponseMsg> outputStream, |
|||
BiConsumer<EdgeId, EdgeGrpcSession> sessionOpenListener, |
|||
BiConsumer<Edge, UUID> sessionCloseListener, ScheduledExecutorService sendDownlinkExecutorService, |
|||
int maxInboundMessageSize, int maxHighPriorityQueueSizePerSession) { |
|||
super(ctx, outputStream, sessionOpenListener, sessionCloseListener, sendDownlinkExecutorService, maxInboundMessageSize, maxHighPriorityQueueSizePerSession); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Boolean> migrateEdgeEvents() { |
|||
return Futures.immediateFuture(Boolean.FALSE); |
|||
} |
|||
|
|||
} |
|||
@ -1,117 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
|
|||
import java.nio.charset.StandardCharsets; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.sql.Connection; |
|||
import java.sql.ResultSet; |
|||
import java.sql.SQLException; |
|||
import java.sql.SQLWarning; |
|||
import java.sql.Statement; |
|||
|
|||
@Slf4j |
|||
public abstract class AbstractSqlTsDatabaseUpgradeService { |
|||
|
|||
protected static final String CALL_REGEX = "call "; |
|||
protected static final String DROP_TABLE = "DROP TABLE "; |
|||
protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS "; |
|||
protected static final String TS_KV_SQL = "ts_kv.sql"; |
|||
protected static final String PATH_TO_USERS_PUBLIC_FOLDER = "C:\\Users\\Public"; |
|||
protected static final String THINGSBOARD_WINDOWS_UPGRADE_DIR = "THINGSBOARD_WINDOWS_UPGRADE_DIR"; |
|||
|
|||
@Value("${spring.datasource.url}") |
|||
protected String dbUrl; |
|||
|
|||
@Value("${spring.datasource.username}") |
|||
protected String dbUserName; |
|||
|
|||
@Value("${spring.datasource.password}") |
|||
protected String dbPassword; |
|||
|
|||
@Autowired |
|||
protected InstallScripts installScripts; |
|||
|
|||
protected abstract void loadSql(Connection conn, String fileName, String version); |
|||
|
|||
protected void loadFunctions(Path sqlFile, Connection conn) throws Exception { |
|||
String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); |
|||
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
} |
|||
|
|||
protected boolean checkVersion(Connection conn) { |
|||
boolean versionValid = false; |
|||
try { |
|||
Statement statement = conn.createStatement(); |
|||
ResultSet resultSet = statement.executeQuery("SELECT current_setting('server_version_num')"); |
|||
resultSet.next(); |
|||
if(resultSet.getLong(1) > 110000) { |
|||
versionValid = true; |
|||
} |
|||
statement.close(); |
|||
} catch (Exception e) { |
|||
log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage()); |
|||
} |
|||
return versionValid; |
|||
} |
|||
|
|||
protected boolean isOldSchema(Connection conn, long fromVersion) { |
|||
boolean isOldSchema = true; |
|||
try { |
|||
Statement statement = conn.createStatement(); |
|||
statement.execute("CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version));"); |
|||
Thread.sleep(1000); |
|||
ResultSet resultSet = statement.executeQuery("SELECT schema_version FROM tb_schema_settings;"); |
|||
if (resultSet.next()) { |
|||
isOldSchema = resultSet.getLong(1) <= fromVersion; |
|||
} else { |
|||
resultSet.close(); |
|||
statement.execute("INSERT INTO tb_schema_settings (schema_version) VALUES (" + fromVersion + ")"); |
|||
} |
|||
statement.close(); |
|||
} catch (InterruptedException | SQLException e) { |
|||
log.info("Failed to check current PostgreSQL schema due to: {}", e.getMessage()); |
|||
} |
|||
return isOldSchema; |
|||
} |
|||
|
|||
protected void executeQuery(Connection conn, String query) { |
|||
try { |
|||
Statement statement = conn.createStatement(); |
|||
statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
|
|||
SQLWarning warnings = statement.getWarnings(); |
|||
if (warnings != null) { |
|||
log.info("{}", warnings.getMessage()); |
|||
SQLWarning nextWarning = warnings.getNextWarning(); |
|||
while (nextWarning != null) { |
|||
log.info("{}", nextWarning.getMessage()); |
|||
nextWarning = nextWarning.getNextWarning(); |
|||
} |
|||
} |
|||
Thread.sleep(2000); |
|||
log.info("Successfully executed query: {}", query); |
|||
} catch (InterruptedException | SQLException e) { |
|||
log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); |
|||
throw new RuntimeException("Failed to execute query:" + query + " due to: ", e); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -1,37 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.NoSqlTsDao; |
|||
|
|||
@Service |
|||
@NoSqlTsDao |
|||
@Profile("install") |
|||
@Slf4j |
|||
public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabaseUpgradeService implements DatabaseTsUpgradeService { |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
switch (fromVersion) { |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,140 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.boot.info.BuildProperties; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.jdbc.core.JdbcTemplate; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.service.install.update.DefaultDataUpdateService; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSettingsService { |
|||
|
|||
private static final String CURRENT_PRODUCT = "CE"; |
|||
private static final List<String> SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.8.0", "3.8.1"); |
|||
|
|||
private final BuildProperties buildProperties; |
|||
private final JdbcTemplate jdbcTemplate; |
|||
|
|||
private String packageSchemaVersion; |
|||
private String schemaVersionFromDb; |
|||
|
|||
@Override |
|||
public void validateSchemaSettings() { |
|||
//TODO: remove after release (3.9.0)
|
|||
createProductIfNotExists(); |
|||
|
|||
String dbSchemaVersion = getDbSchemaVersion(); |
|||
|
|||
if (DefaultDataUpdateService.getEnv("SKIP_SCHEMA_VERSION_CHECK", false)) { |
|||
log.info("Skipped DB schema version check due to SKIP_SCHEMA_VERSION_CHECK set to 'true'."); |
|||
return; |
|||
} |
|||
|
|||
String product = getProductFromDb(); |
|||
if (!CURRENT_PRODUCT.equals(product)) { |
|||
onSchemaSettingsError(String.format("Upgrade failed: can't upgrade ThingsBoard %s database using ThingsBoard %s.", product, CURRENT_PRODUCT)); |
|||
} |
|||
|
|||
if (dbSchemaVersion.equals(getPackageSchemaVersion())) { |
|||
onSchemaSettingsError("Upgrade failed: database already upgraded to current version. You can set SKIP_SCHEMA_VERSION_CHECK to 'true' if force re-upgrade needed."); |
|||
} |
|||
|
|||
if (!SUPPORTED_VERSIONS_FOR_UPGRADE.contains(dbSchemaVersion)) { |
|||
onSchemaSettingsError(String.format("Upgrade failed: database version '%s' is not supported for upgrade. Supported versions are: %s.", |
|||
dbSchemaVersion, SUPPORTED_VERSIONS_FOR_UPGRADE |
|||
)); |
|||
} |
|||
} |
|||
|
|||
@Deprecated(forRemoval = true, since = "3.9.0") |
|||
private void createProductIfNotExists() { |
|||
boolean isCommunityEdition = jdbcTemplate.queryForList( |
|||
"SELECT 1 FROM information_schema.tables WHERE table_name = 'integration'", Integer.class).isEmpty(); |
|||
String product = isCommunityEdition ? "CE" : "PE"; |
|||
jdbcTemplate.execute("ALTER TABLE tb_schema_settings ADD COLUMN IF NOT EXISTS product varchar(2) DEFAULT '" + product + "'"); |
|||
} |
|||
|
|||
@Override |
|||
public void createSchemaSettings() { |
|||
Long schemaVersion = getSchemaVersionFromDb(); |
|||
if (schemaVersion == null) { |
|||
jdbcTemplate.execute("INSERT INTO tb_schema_settings (schema_version, product) VALUES (" + getPackageSchemaVersionForDb() + ", '" + CURRENT_PRODUCT + "')"); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void updateSchemaVersion() { |
|||
jdbcTemplate.execute("UPDATE tb_schema_settings SET schema_version = " + getPackageSchemaVersionForDb()); |
|||
} |
|||
|
|||
@Override |
|||
public String getPackageSchemaVersion() { |
|||
if (packageSchemaVersion == null) { |
|||
packageSchemaVersion = buildProperties.getVersion().replaceAll("[^\\d.]", ""); |
|||
} |
|||
return packageSchemaVersion; |
|||
} |
|||
|
|||
@Override |
|||
public String getDbSchemaVersion() { |
|||
if (schemaVersionFromDb == null) { |
|||
Long version = getSchemaVersionFromDb(); |
|||
if (version == null) { |
|||
onSchemaSettingsError("Upgrade failed: the database schema version is missing."); |
|||
} |
|||
|
|||
@SuppressWarnings("DataFlowIssue") |
|||
long major = version / 1000000; |
|||
long minor = (version % 1000000) / 1000; |
|||
long patch = version % 1000; |
|||
|
|||
schemaVersionFromDb = major + "." + minor + "." + patch; |
|||
} |
|||
return schemaVersionFromDb; |
|||
} |
|||
|
|||
private Long getSchemaVersionFromDb() { |
|||
return jdbcTemplate.queryForList("SELECT schema_version FROM tb_schema_settings", Long.class).stream().findFirst().orElse(null); |
|||
} |
|||
|
|||
private String getProductFromDb() { |
|||
return jdbcTemplate.queryForList("SELECT product FROM tb_schema_settings", String.class).stream().findFirst().orElse(null); |
|||
} |
|||
|
|||
private long getPackageSchemaVersionForDb() { |
|||
String[] versionParts = getPackageSchemaVersion().split("\\."); |
|||
|
|||
long major = Integer.parseInt(versionParts[0]); |
|||
long minor = Integer.parseInt(versionParts[1]); |
|||
long patch = versionParts.length > 2 ? Integer.parseInt(versionParts[2]) : 0; |
|||
|
|||
return major * 1000000 + minor * 1000 + patch; |
|||
} |
|||
|
|||
private void onSchemaSettingsError(String message) { |
|||
Runtime.getRuntime().addShutdownHook(new Thread(() -> log.error(message))); |
|||
throw new RuntimeException(message); |
|||
} |
|||
} |
|||
@ -1,51 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.SqlTsDao; |
|||
|
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
import java.sql.Connection; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@Slf4j |
|||
@SqlTsDao |
|||
public class SqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
switch (fromVersion) { |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
protected void loadSql(Connection conn, String fileName, String version) { |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, fileName); |
|||
try { |
|||
loadFunctions(schemaUpdateFile, conn); |
|||
log.info("Functions successfully loaded!"); |
|||
} catch (Exception e) { |
|||
log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); |
|||
} |
|||
} |
|||
} |
|||
@ -1,55 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.install; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.context.annotation.Profile; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.dao.util.TimescaleDBTsDao; |
|||
|
|||
import java.nio.file.Path; |
|||
import java.nio.file.Paths; |
|||
import java.sql.Connection; |
|||
|
|||
@Service |
|||
@Profile("install") |
|||
@Slf4j |
|||
@TimescaleDBTsDao |
|||
public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService { |
|||
|
|||
@Autowired |
|||
private InstallScripts installScripts; |
|||
|
|||
@Override |
|||
public void upgradeDatabase(String fromVersion) throws Exception { |
|||
switch (fromVersion) { |
|||
default: |
|||
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
protected void loadSql(Connection conn, String fileName, String version) { |
|||
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, fileName); |
|||
try { |
|||
loadFunctions(schemaUpdateFile, conn); |
|||
log.info("Functions successfully loaded!"); |
|||
} catch (Exception e) { |
|||
log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage()); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,160 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.ttl; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.scheduling.annotation.Scheduled; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.edge.Edge; |
|||
import org.thingsboard.server.common.data.id.EdgeId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.edge.EdgeService; |
|||
import org.thingsboard.server.dao.tenant.TenantService; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.TopicService; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaAdmin; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaSettings; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
|
|||
import java.time.Instant; |
|||
import java.util.ArrayList; |
|||
import java.util.Date; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_CONNECT_TIME; |
|||
|
|||
@Slf4j |
|||
@Service |
|||
@TbCoreComponent |
|||
@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ${edges.enabled:true} && ${sql.ttl.edge_events.edge_events_ttl:0} > 0") |
|||
public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { |
|||
|
|||
private static final String EDGE_EVENT_TOPIC_NAME = "tb_edge_event.notifications."; |
|||
|
|||
private final TopicService topicService; |
|||
private final TenantService tenantService; |
|||
private final EdgeService edgeService; |
|||
private final AttributesService attributesService; |
|||
private final TbKafkaAdmin kafkaAdmin; |
|||
|
|||
@Value("${sql.ttl.edge_events.edge_events_ttl:2628000}") |
|||
private long ttlSeconds; |
|||
|
|||
public KafkaEdgeTopicsCleanUpService(PartitionService partitionService, EdgeService edgeService, |
|||
TenantService tenantService, AttributesService attributesService, |
|||
TopicService topicService, TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { |
|||
super(partitionService); |
|||
this.topicService = topicService; |
|||
this.tenantService = tenantService; |
|||
this.edgeService = edgeService; |
|||
this.attributesService = attributesService; |
|||
this.kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); |
|||
} |
|||
|
|||
@Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.edge_events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}") |
|||
public void cleanUp() { |
|||
if (!isSystemTenantPartitionMine()) { |
|||
return; |
|||
} |
|||
|
|||
Set<String> topics = kafkaAdmin.getAllTopics(); |
|||
if (topics == null || topics.isEmpty()) { |
|||
return; |
|||
} |
|||
|
|||
String edgeTopicPrefix = topicService.buildTopicName(EDGE_EVENT_TOPIC_NAME); |
|||
List<String> matchingTopics = topics.stream().filter(topic -> topic.startsWith(edgeTopicPrefix)).toList(); |
|||
if (matchingTopics.isEmpty()) { |
|||
log.debug("No matching topics found with prefix [{}]. Skipping cleanup.", edgeTopicPrefix); |
|||
return; |
|||
} |
|||
|
|||
Map<TenantId, List<EdgeId>> tenantEdgeMap = extractTenantAndEdgeIds(matchingTopics, edgeTopicPrefix); |
|||
|
|||
long currentTimeMillis = System.currentTimeMillis(); |
|||
long ttlMillis = TimeUnit.SECONDS.toMillis(ttlSeconds); |
|||
|
|||
tenantEdgeMap.forEach((tenantId, edgeIds) -> processTenantCleanUp(tenantId, edgeIds, ttlMillis, currentTimeMillis)); |
|||
} |
|||
|
|||
private void processTenantCleanUp(TenantId tenantId, List<EdgeId> edgeIds, long ttlMillis, long currentTimeMillis) { |
|||
boolean tenantExists = tenantService.tenantExists(tenantId); |
|||
if (tenantExists) { |
|||
for (EdgeId edgeId : edgeIds) { |
|||
try { |
|||
attributesService.find(tenantId, edgeId, AttributeScope.SERVER_SCOPE, LAST_CONNECT_TIME).get() |
|||
.flatMap(AttributeKvEntry::getLongValue) |
|||
.filter(lastConnectTime -> isTopicExpired(lastConnectTime, ttlMillis, currentTimeMillis)) |
|||
.ifPresentOrElse(lastConnectTime -> { |
|||
String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); |
|||
if (kafkaAdmin.isTopicEmpty(topic)) { |
|||
kafkaAdmin.deleteTopic(topic); |
|||
log.info("[{}] Removed outdated topic {} for edge {} older than {}", |
|||
tenantId, topic, edgeId, Date.from(Instant.ofEpochMilli(currentTimeMillis - ttlMillis))); |
|||
} |
|||
}, () -> { |
|||
Edge edge = edgeService.findEdgeById(tenantId, edgeId); |
|||
if (edge == null) { |
|||
String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); |
|||
kafkaAdmin.deleteTopic(topic); |
|||
log.info("[{}] Removed topic {} for deleted edge {}", tenantId, topic, edgeId); |
|||
} |
|||
}); |
|||
} catch (Exception e) { |
|||
log.error("[{}] Failed to delete topic for edge {}", tenantId, edgeId, e); |
|||
} |
|||
} |
|||
} else { |
|||
for (EdgeId edgeId : edgeIds) { |
|||
String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); |
|||
kafkaAdmin.deleteTopic(topic); |
|||
} |
|||
log.info("[{}] Removed topics for not existing tenant and edges {}", tenantId, edgeIds); |
|||
} |
|||
} |
|||
|
|||
private boolean isTopicExpired(long lastConnectTime, long ttlMillis, long currentTimeMillis) { |
|||
return lastConnectTime + ttlMillis < currentTimeMillis; |
|||
} |
|||
|
|||
private Map<TenantId, List<EdgeId>> extractTenantAndEdgeIds(List<String> topics, String prefix) { |
|||
Map<TenantId, List<EdgeId>> tenantEdgeMap = new HashMap<>(); |
|||
for (String topic : topics) { |
|||
try { |
|||
String remaining = topic.substring(prefix.length()); |
|||
String[] parts = remaining.split("\\."); |
|||
TenantId tenantId = new TenantId(UUID.fromString(parts[0])); |
|||
EdgeId edgeId = new EdgeId(UUID.fromString(parts[1])); |
|||
tenantEdgeMap.computeIfAbsent(tenantId, id -> new ArrayList<>()).add(edgeId); |
|||
} catch (Exception e) { |
|||
log.warn("Failed to extract TenantId and EdgeId from topic [{}]", topic, e); |
|||
} |
|||
} |
|||
return tenantEdgeMap; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,157 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.transport.mqtt.mqttv5.rpc; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.fasterxml.jackson.databind.node.NullNode; |
|||
import io.netty.handler.codec.mqtt.MqttQoS; |
|||
import lombok.Getter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.eclipse.paho.mqttv5.client.IMqttToken; |
|||
import org.eclipse.paho.mqttv5.client.MqttDisconnectResponse; |
|||
import org.eclipse.paho.mqttv5.common.MqttException; |
|||
import org.eclipse.paho.mqttv5.common.MqttMessage; |
|||
import org.eclipse.paho.mqttv5.common.packet.MqttProperties; |
|||
import org.eclipse.paho.mqttv5.common.packet.MqttReturnCode; |
|||
import org.eclipse.paho.mqttv5.common.packet.MqttWireMessage; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
import org.springframework.test.context.TestPropertySource; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.rpc.Rpc; |
|||
import org.thingsboard.server.common.data.rpc.RpcStatus; |
|||
import org.thingsboard.server.dao.service.DaoSqlTest; |
|||
import org.thingsboard.server.transport.mqtt.MqttTestConfigProperties; |
|||
import org.thingsboard.server.transport.mqtt.mqttv5.MqttV5TestCallback; |
|||
import org.thingsboard.server.transport.mqtt.mqttv5.MqttV5TestClient; |
|||
|
|||
import java.util.concurrent.CountDownLatch; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; |
|||
import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_RPC_REQUESTS_SUB_TOPIC; |
|||
|
|||
@Slf4j |
|||
@DaoSqlTest |
|||
@TestPropertySource(properties = { |
|||
"actors.rpc.close_session_on_rpc_delivery_timeout=true", |
|||
"transport.mqtt.timeout=100", |
|||
}) |
|||
public class MqttV5CloseTransportSessionOnRpcDeliveryTimeoutIntegrationTest extends AbstractMqttV5RpcTest { |
|||
|
|||
@Before |
|||
public void beforeTest() throws Exception { |
|||
MqttTestConfigProperties configProperties = MqttTestConfigProperties.builder() |
|||
.deviceName("RPC test device") |
|||
.build(); |
|||
processBeforeTest(configProperties); |
|||
} |
|||
|
|||
@Test |
|||
public void testOneWayRpcCloseSessionOnRpcDeliveryTimeout() throws Exception { |
|||
testCloseSessionOnRpcDeliveryTimeout("oneway"); |
|||
} |
|||
|
|||
@Test |
|||
public void testTwoWayRpcCloseSessionOnRpcDeliveryTimeout() throws Exception { |
|||
testCloseSessionOnRpcDeliveryTimeout("twoway"); |
|||
} |
|||
|
|||
private void testCloseSessionOnRpcDeliveryTimeout(String rpcType) throws Exception { |
|||
MqttV5TestClient client = new MqttV5TestClient(); |
|||
client.enableManualAcks(); |
|||
client.connectAndWait(accessToken); |
|||
MqttV5NoAckTestCallback callback = new MqttV5NoAckTestCallback(); |
|||
client.setCallback(callback); |
|||
client.subscribeAndWait(DEVICE_RPC_REQUESTS_SUB_TOPIC, MqttQoS.AT_LEAST_ONCE); |
|||
|
|||
String expectedReceivedPayload = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; |
|||
long expirationTime = System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1); |
|||
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1},\"persistent\":true,\"retries\":0,\"expirationTime\": " + expirationTime + "}"; |
|||
String result = doPostAsync("/api/rpc/" + rpcType + "/" + savedDevice.getId(), setGpioRequest, String.class, status().isOk()); |
|||
|
|||
assertThat(result).isNotNull(); |
|||
JsonNode response = JacksonUtil.toJsonNode(result); |
|||
assertThat(response).isNotNull(); |
|||
assertThat(response.hasNonNull("rpcId")).isTrue(); |
|||
|
|||
callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); |
|||
assertThat(callback.getQoS()).isEqualTo(MqttQoS.AT_LEAST_ONCE.value()); |
|||
assertThat(JacksonUtil.fromBytes(callback.getPayloadBytes())) |
|||
.isEqualTo(JacksonUtil.toJsonNode(expectedReceivedPayload)); |
|||
|
|||
callback.getDisconnectLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); |
|||
assertThat(callback.getReturnCode()).isEqualTo(MqttReturnCode.RETURN_CODE_ADMINISTRITIVE_ACTION); |
|||
|
|||
Rpc persistedRpc = doGet("/api/rpc/persistent/" + response.get("rpcId").asText(), Rpc.class); |
|||
assertThat(persistedRpc).isNotNull(); |
|||
assertThat(persistedRpc.getStatus()).isEqualTo(RpcStatus.QUEUED); |
|||
assertThat(persistedRpc.getResponse()).isInstanceOf(NullNode.class); |
|||
assertThat(client.isConnected()).isFalse(); |
|||
} |
|||
|
|||
@Getter |
|||
private static class MqttV5NoAckTestCallback extends MqttV5TestCallback { |
|||
|
|||
private int returnCode; |
|||
|
|||
public CountDownLatch getDisconnectLatch() { |
|||
return super.getDeliveryLatch(); |
|||
} |
|||
|
|||
@Override |
|||
public void disconnected(MqttDisconnectResponse mqttDisconnectResponse) { |
|||
log.warn("MqttDisconnectResponse: {}", mqttDisconnectResponse); |
|||
returnCode = mqttDisconnectResponse.getReturnCode(); |
|||
getDisconnectLatch().countDown(); |
|||
} |
|||
|
|||
@Override |
|||
public void mqttErrorOccurred(MqttException e) { |
|||
log.warn("Error occurred:", e); |
|||
} |
|||
|
|||
@Override |
|||
public void messageArrived(String requestTopic, MqttMessage mqttMessage) { |
|||
log.warn("messageArrived on topic: {}", requestTopic); |
|||
qoS = mqttMessage.getQos(); |
|||
payloadBytes = mqttMessage.getPayload(); |
|||
subscribeLatch.countDown(); |
|||
} |
|||
|
|||
@Override |
|||
public void deliveryComplete(IMqttToken iMqttToken) { |
|||
// should be never called, Since we're never going to send a response back to server.
|
|||
log.warn("delivery complete: {}", iMqttToken.getResponse()); |
|||
pubAckReceived = iMqttToken.getResponse().getType() == MqttWireMessage.MESSAGE_TYPE_PUBACK; |
|||
getDisconnectLatch().countDown(); |
|||
} |
|||
|
|||
@Override |
|||
public void connectComplete(boolean reconnect, String serverURI) { |
|||
log.warn("Connect completed: reconnect - {}, serverURI - {}", reconnect, serverURI); |
|||
} |
|||
|
|||
@Override |
|||
public void authPacketArrived(int reasonCode, MqttProperties mqttProperties) { |
|||
log.warn("Auth package received: reasonCode - {}, mqtt properties - {}", reasonCode, mqttProperties); |
|||
} |
|||
|
|||
|
|||
} |
|||
|
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue