From b30bbf9d80183c3839ffcca8bf6df2c0dc609098 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 14 Dec 2023 19:05:54 +0200 Subject: [PATCH 01/17] attribute_kv optimization: deleted entity_type column, updated attribute_type and attribute_key columns --- .../3.6.2/schema_update_attribute_kv.sql | 181 ++++++++++++++++++ .../data/upgrade/3.6.2/schema_update_ttl.sql | 87 +++++++++ .../install/ThingsboardInstallService.java | 6 + .../CassandraTsDatabaseUpgradeService.java | 1 + .../install/SqlDatabaseUpgradeService.java | 97 ++++++++++ .../install/SqlTsDatabaseUpgradeService.java | 2 + .../TimescaleTsDatabaseUpgradeService.java | 5 + .../query/DefaultEntityQueryService.java | 11 +- .../dao/attributes/AttributesService.java | 3 +- .../server/common/data/AttributeScope.java | 32 ++++ .../server/dao/attributes/AttributeUtils.java | 3 +- .../server/dao/attributes/AttributesDao.java | 14 +- .../dao/attributes/BaseAttributesService.java | 18 +- .../attributes/CachedAttributesService.java | 18 +- .../model/sql/AttributeKvCompositeKey.java | 11 +- .../dao/model/sql/AttributeKvDictionary.java | 41 ++++ .../dao/model/sql/AttributeKvEntity.java | 14 +- .../server/dao/service/Validator.java | 14 ++ .../AttributeKvDictionaryRepository.java | 29 +++ .../AttributeKvInsertRepository.java | 58 +++--- .../sql/attributes/AttributeKvRepository.java | 29 ++- .../dao/sql/attributes/JpaAttributeDao.java | 113 ++++++++--- .../dao/sql/query/EntityKeyMapping.java | 20 +- .../main/resources/sql/schema-entities.sql | 13 +- .../main/resources/sql/schema-timescale.sql | 4 +- dao/src/main/resources/sql/schema-ts-psql.sql | 4 +- .../sql/schema-views-and-functions.sql | 2 +- .../profile/TbDeviceProfileNodeTest.java | 58 ++++-- 28 files changed, 729 insertions(+), 159 deletions(-) create mode 100644 application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql create mode 100644 application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionary.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java diff --git a/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql b/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql new file mode 100644 index 0000000000..142b3c1844 --- /dev/null +++ b/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql @@ -0,0 +1,181 @@ +-- +-- Copyright © 2016-2023 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + + +-- create new attribute_kv table schema +DO +$$ + BEGIN + -- in case of running the upgrade script a second time: + IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'attribute_kv' and column_name='entity_type') THEN + IF EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_attribute_kv_by_key_and_last_update_ts') THEN + ALTER INDEX idx_attribute_kv_by_key_and_last_update_ts RENAME TO idx_attribute_kv_by_key_and_last_update_ts_old; + END IF; + IF EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'attribute_kv_pkey') THEN + ALTER TABLE attribute_kv RENAME CONSTRAINT attribute_kv_pkey TO attribute_kv_pkey_old; + END IF; + ALTER TABLE attribute_kv + RENAME TO attribute_kv_old; + CREATE TABLE IF NOT EXISTS attribute_kv + ( + entity_id uuid, + attribute_type int, + attribute_key int, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json, + last_update_ts bigint, + CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) + ); + END IF; + END; +$$; + +-- create attribute_kv_dictionary table +CREATE TABLE IF NOT EXISTS attribute_kv_dictionary +( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT attribute_key_id_pkey PRIMARY KEY (key) +); + +-- create to_attribute_type_id +CREATE OR REPLACE FUNCTION to_attribute_type_id(IN attribute_type varchar, OUT attribute_type_id int) AS +$$ +BEGIN + CASE + WHEN attribute_type = 'CLIENT_SCOPE' THEN + attribute_type_id := 1; + WHEN attribute_type = 'SERVER_SCOPE' THEN + attribute_type_id := 2; + WHEN attribute_type = 'SHARED_SCOPE' THEN + attribute_type_id := 3; + END CASE; +END; +$$ LANGUAGE plpgsql; + + +-- insert keys into attribute_kv_dictionary +DO +$$ +DECLARE + insert_record RECORD; + key_cursor refcursor; +BEGIN + IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN + OPEN key_cursor FOR SELECT DISTINCT attribute_key + FROM attribute_kv_old + ORDER BY attribute_key; + LOOP + FETCH key_cursor INTO insert_record; + EXIT WHEN NOT FOUND; + IF NOT EXISTS(SELECT key FROM attribute_kv_dictionary WHERE key = insert_record.attribute_key) THEN + INSERT INTO attribute_kv_dictionary(key) VALUES (insert_record.attribute_key); + END IF; + END LOOP; + CLOSE key_cursor; + END IF; +END; +$$; + +-- create procedure to migrate all rows from attribute_kv_old to attribute_kv +CREATE OR REPLACE PROCEDURE insert_into_attribute_kv(IN path_to_file varchar) + LANGUAGE plpgsql AS +$$ +DECLARE + row_num_old integer; + row_num integer; + attribute_scope_array text[]; +BEGIN + attribute_scope_array := ARRAY['SERVER_SCOPE', 'CLIENT_SCOPE', 'SHARED_SCOPE']; + IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN + EXECUTE format('COPY (SELECT records.entity_id AS entity_id, + to_attribute_type_id(records.attribute_type) AS attribute_type, + records.attribute_key AS attribute_key, + records.bool_v AS bool_v, + records.str_v AS str_v, + records.long_v AS long_v, + records.dbl_v AS dbl_v, + records.json_v AS json_v, + records.last_update_ts AS last_update_ts + FROM (SELECT entity_id, + attribute_type, + key_id AS attribute_key, + bool_v, + str_v, + long_v, + dbl_v, + json_v, + last_update_ts + FROM attribute_kv_old INNER JOIN attribute_kv_dictionary ON (attribute_kv_old.attribute_key = attribute_kv_dictionary.key) + WHERE attribute_type= ANY(%L)) AS records) TO %L;', attribute_scope_array, path_to_file); + EXECUTE format('COPY attribute_kv FROM %L', path_to_file); + SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; + SELECT COUNT(*) INTO row_num FROM attribute_kv; + RAISE NOTICE 'Migrated % of % rows', row_num, row_num_old; + END IF; +EXCEPTION + WHEN others THEN + ROLLBACK; + RAISE EXCEPTION 'Error during COPY: %', SQLERRM; +END +$$; + +CREATE OR REPLACE PROCEDURE recreate_device_info_active_attribute_view() + LANGUAGE plpgsql AS +$$ +BEGIN + DROP VIEW IF EXISTS device_info_active_attribute_view CASCADE; + CREATE OR REPLACE VIEW device_info_active_attribute_view AS + SELECT d.* + , c.title as customer_title + , COALESCE((c.additional_info::json->>'isPublic')::bool, FALSE) as customer_is_public + , d.type as device_profile_name + , COALESCE(da.bool_v, FALSE) as active + FROM device d + LEFT JOIN customer c ON c.id = d.customer_id + LEFT JOIN attribute_kv da ON da.entity_id = d.id AND da.attribute_type = 2 AND da.attribute_key = (select key_id from attribute_kv_dictionary where key = 'active'); +END; +$$; + +CREATE OR REPLACE PROCEDURE recreate_device_info_view() + LANGUAGE plpgsql AS +$$ +BEGIN + DROP VIEW IF EXISTS device_info_view CASCADE; + CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view; +END; +$$; + +CREATE OR REPLACE PROCEDURE drop_attribute_kv_old_table() + LANGUAGE plpgsql AS +$$ +DECLARE + row_num integer; +BEGIN + SELECT COUNT(*) INTO row_num FROM attribute_kv; + IF row_num != 0 then + DROP TABLE IF EXISTS attribute_kv_old; + DROP PROCEDURE IF EXISTS insert_into_attribute_kv(IN path_to_file varchar); + ELSE + RAISE EXCEPTION 'Table attribute_kv is empty'; + END IF; + RETURN; +END; +$$; + diff --git a/application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql b/application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql new file mode 100644 index 0000000000..67ec25656e --- /dev/null +++ b/application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql @@ -0,0 +1,87 @@ +-- +-- Copyright © 2016-2023 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE +tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; +tenant_id_record uuid; + customer_id_record uuid; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN +OPEN tenant_cursor; +FETCH tenant_cursor INTO tenant_id_record; +WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; +END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; +END IF; +FOR customer_id_record IN +SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + customer_id_record, 'TTL') INTO customer_ttl; +IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; +ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; +END IF; +END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; +END IF; +END LOOP; +FETCH tenant_cursor INTO tenant_id_record; +END LOOP; +END +$$; \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 3829b3cf45..ac348c1249 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -265,6 +265,12 @@ public class ThingsboardInstallService { case "3.6.0": log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); + case "3.6.2": + log.info("Upgrading ThingsBoard from version 3.6.2 to 3.7.0 ..."); + if (databaseTsUpgradeService != null) { + databaseTsUpgradeService.upgradeDatabase("3.6.2"); + } + databaseEntitiesUpgradeService.upgradeDatabase("3.6.2"); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java index 5c7c51eb2c..3e2f29440c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java @@ -52,6 +52,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase case "3.1.1": case "3.2.1": case "3.2.2": + case "3.6.2": break; default: throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index c02b7e9cc1..2983dea649 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -19,12 +19,14 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.SystemUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -48,6 +50,8 @@ import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; import org.thingsboard.server.service.install.sql.SqlDbHelper; import org.thingsboard.server.service.install.update.DefaultDataUpdateService; +import java.io.File; +import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -64,6 +68,8 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; +import static org.thingsboard.server.service.install.AbstractSqlTsDatabaseUpgradeService.PATH_TO_USERS_PUBLIC_FOLDER; +import static org.thingsboard.server.service.install.AbstractSqlTsDatabaseUpgradeService.THINGSBOARD_WINDOWS_UPGRADE_DIR; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; import static org.thingsboard.server.service.install.DatabaseHelper.ASSIGNED_CUSTOMERS; import static org.thingsboard.server.service.install.DatabaseHelper.CONFIGURATION; @@ -89,6 +95,7 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { private static final String SCHEMA_UPDATE_SQL = "schema_update.sql"; + private static final String LOAD_ATTRIBUTE_KV_FUNCTIONS_SQL = "schema_update_attribute_kv.sql"; @Value("${spring.datasource.url}") private String dbUrl; @@ -792,11 +799,88 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.6.2": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + if (isOldSchema(conn, 3006002)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.6.2", LOAD_ATTRIBUTE_KV_FUNCTIONS_SQL); + loadSql(schemaUpdateFile, conn); + + Path pathToTempAttributeKvFile; + if (SystemUtils.IS_OS_WINDOWS) { + pathToTempAttributeKvFile = createTempFileWindows("attribute_kv_temp",".sql"); + executeQuery(conn, "call insert_into_attribute_kv('" + pathToTempAttributeKvFile + "')"); + } else { + pathToTempAttributeKvFile = createTempFile("attribute_kv", "attribute_kv_temp.sql"); + executeQuery(conn, "call insert_into_attribute_kv('" + pathToTempAttributeKvFile + "')"); + } + + //recreate device_info_active_attribute_view + executeQuery(conn, "call recreate_device_info_active_attribute_view()"); + executeQuery(conn, "call recreate_device_info_view()"); + + // remove attribute_kv_old + executeQuery(conn, "call drop_attribute_kv_old_table()"); + + //create index for new table attribute_kv + executeQuery(conn, "CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);"); + + // remove temp files + if (pathToTempAttributeKvFile.toFile().exists()) { + boolean deleteTsKvFile = pathToTempAttributeKvFile.toFile().delete(); + if (deleteTsKvFile) { + log.info("Successfully deleted the temp file for attribute_kv table upgrade!"); + } + } + + executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 3007000;"); + log.info("Schema updated to version 3.7.0."); + } else { + log.info("Skip schema re-update to version 3.7.0. Use env flag 'SKIP_SCHEMA_VERSION_CHECK' to force the re-update."); + } + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } } + private static Path createTempFile(String tempDirectoryName, String tempFileName) throws IOException { + Path pathToTempAttributeKvFile; + Path tempDirPath = Files.createTempDirectory(tempDirectoryName); + File tempDirAsFile = tempDirPath.toFile(); + boolean writable = tempDirAsFile.setWritable(true, false); + boolean readable = tempDirAsFile.setReadable(true, false); + boolean executable = tempDirAsFile.setExecutable(true, false); + pathToTempAttributeKvFile = tempDirPath.resolve(tempFileName).toAbsolutePath(); + + if (!(writable && readable && executable)) { + throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!"); + } + return pathToTempAttributeKvFile; + } + + private static Path createTempFileWindows(String prefix, String suffix) throws IOException { + Path pathToTempAttributeKvFile; + log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR); + Path pathToDir; + String thingsboardWindowsUpgradeDir = System.getenv("THINGSBOARD_WINDOWS_UPGRADE_DIR"); + if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) { + log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR); + pathToDir = Paths.get(thingsboardWindowsUpgradeDir); + } else { + log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR); + pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER); + } + log.info("Directory: {} will be used for creation temporary upgrade files!", pathToDir); + + Path attributeKvFile = Files.createTempFile(pathToDir, prefix, suffix); + pathToTempAttributeKvFile = attributeKvFile.toAbsolutePath(); + return pathToTempAttributeKvFile; + } + private void runSchemaUpdateScript(Connection connection, String version) throws Exception { Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, SCHEMA_UPDATE_SQL); loadSql(schemaUpdateFile, connection); @@ -811,6 +895,19 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService Thread.sleep(5000); } + private void executeQuery(Connection conn, String query) { + try { + Statement statement = conn.createStatement(); + statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script + printWarnings(statement); + Thread.sleep(2000); + log.info("Successfully executed query: {}", query); + } catch (InterruptedException | SQLException e) { + log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); + throw new RuntimeException("Failed to execute query:" + query + " due to: ", e); + } + } + protected void printWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java index 6400463e11..b61da45631 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java @@ -213,6 +213,8 @@ public class SqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSer loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3"); } break; + case "3.6.2": + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 205ec14591..4477bef1c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -184,6 +184,11 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr break; case "3.2.2": break; + case "3.6.2": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL, "3.6.2"); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index 89b4002223..296c992ccf 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -251,14 +251,13 @@ public class DefaultEntityQueryService implements EntityQueryService { } if (isAttributes) { - Map> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType)); - List>> futures = new ArrayList<>(typesMap.size()); - typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, type, entityIds)))); - attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> { - if (CollectionUtils.isEmpty(lists)) { + ListenableFuture> future; + future = dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, ids)); + attributesKeysFuture = Futures.transform(future, list -> { + if (CollectionUtils.isEmpty(list)) { return Collections.emptyList(); } - return lists.stream().flatMap(List::stream).distinct().sorted().collect(Collectors.toList()); + return list.stream().distinct().sorted().collect(Collectors.toList()); }, dbCallbackExecutor); } else { attributesKeysFuture = null; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 8fab864cc5..de1386d0c6 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -45,6 +44,6 @@ public interface AttributesService { List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); - List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds); + List findAllKeysByEntityIds(TenantId tenantId, List entityIds); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java b/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java new file mode 100644 index 0000000000..83f5210427 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Getter; + +public enum AttributeScope { + + CLIENT_SCOPE(1), + SERVER_SCOPE(2), + SHARED_SCOPE(3); + @Getter + private final int id; + + AttributeScope(int id) { + this.id = id; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java index 192d56334d..29c44edd68 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.attributes; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.dao.exception.IncorrectParameterException; @@ -27,7 +28,7 @@ public class AttributeUtils { public static void validate(EntityId id, String scope) { Validator.validateId(id.getId(), "Incorrect id " + id); - Validator.validateString(scope, "Incorrect scope " + scope); + Validator.validateEnum(AttributeScope.class, scope, "Incorrect scope " + scope); } public static void validate(List kvEntries, boolean valueNoXssValidation) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index ffb3b9c229..2f53b0afa7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -16,7 +16,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -31,17 +31,17 @@ import java.util.Optional; */ public interface AttributesDao { - Optional find(TenantId tenantId, EntityId entityId, String attributeType, String attributeKey); + Optional find(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, String attributeKey); - List find(TenantId tenantId, EntityId entityId, String attributeType, Collection attributeKey); + List find(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, Collection attributeKey); - List findAll(TenantId tenantId, EntityId entityId, String attributeType); + List findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope); - ListenableFuture save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute); - List> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List keys); + List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); - List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds); + List findAllKeysByEntityIds(TenantId tenantId, List entityIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index f855c116e2..2defbd9398 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -22,7 +22,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -57,20 +57,20 @@ public class BaseAttributesService implements AttributesService { public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { validate(entityId, scope); Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey); - return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKey)); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey)); } @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { validate(entityId, scope); attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); - return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKeys)); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys)); } @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { validate(entityId, scope); - return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, AttributeScope.valueOf(scope))); } @Override @@ -79,28 +79,28 @@ public class BaseAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { - return attributesDao.findAllKeysByEntityIds(tenantId, entityType, entityIds); + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { + return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, scope, attribute); + return attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); } @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { validate(entityId, scope); - return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); + return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 322d725ff3..327f6b5706 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -26,7 +26,7 @@ import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.TbTransactionalCache; -import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -120,7 +120,7 @@ public class CachedAttributesService implements AttributesService { return cacheExecutor.submit(() -> { var cacheTransaction = cache.newTransactionForKey(attributeCacheKey); try { - Optional result = attributesDao.find(tenantId, entityId, scope, attributeKey); + Optional result = attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey); cacheTransaction.putIfAbsent(attributeCacheKey, result.orElse(null)); cacheTransaction.commit(); return result; @@ -159,7 +159,7 @@ public class CachedAttributesService implements AttributesService { var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); try { log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); - List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); + List result = attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), notFoundAttributeKeys); for (AttributeKvEntry foundInDbAttribute : result) { AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); @@ -198,7 +198,7 @@ public class CachedAttributesService implements AttributesService { @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { validate(entityId, scope); - return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, AttributeScope.valueOf(scope))); } @Override @@ -207,15 +207,15 @@ public class CachedAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { - return attributesDao.findAllKeysByEntityIds(tenantId, entityType, entityIds); + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { + return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + ListenableFuture future = attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); return Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor); } @@ -226,7 +226,7 @@ public class CachedAttributesService implements AttributesService { List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + ListenableFuture future = attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); futures.add(Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor)); } @@ -243,7 +243,7 @@ public class CachedAttributesService implements AttributesService { @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { validate(entityId, scope); - List> futures = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); + List> futures = attributesDao.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, key -> { cache.evict(new AttributeCacheKey(scope, entityId, key)); return key; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvCompositeKey.java index 936a18eeeb..80be913468 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvCompositeKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvCompositeKey.java @@ -17,12 +17,9 @@ package org.thingsboard.server.dao.model.sql; import jakarta.persistence.Column; import jakarta.persistence.Embeddable; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.EntityType; import java.io.Serializable; import java.util.UUID; @@ -30,20 +27,16 @@ import java.util.UUID; import static org.thingsboard.server.dao.model.ModelConstants.ATTRIBUTE_KEY_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.ATTRIBUTE_TYPE_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN; @Data @AllArgsConstructor @NoArgsConstructor @Embeddable public class AttributeKvCompositeKey implements Serializable { - @Enumerated(EnumType.STRING) - @Column(name = ENTITY_TYPE_COLUMN) - private EntityType entityType; @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid") private UUID entityId; @Column(name = ATTRIBUTE_TYPE_COLUMN) - private String attributeType; + private int attributeType; @Column(name = ATTRIBUTE_KEY_COLUMN) - private String attributeKey; + private int attributeKey; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionary.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionary.java new file mode 100644 index 0000000000..1cde08912e --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionary.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Data; +import org.hibernate.annotations.Generated; + +import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; + +@Data +@Entity +@Table(name = "attribute_kv_dictionary") +public final class AttributeKvDictionary { + + @Id + @Column(name = KEY_COLUMN) + private String key; + + @Column(name = KEY_ID_COLUMN, unique = true, columnDefinition="int") + @Generated + private int keyId; + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java index bc97c0394f..1bb23aaa89 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java @@ -30,6 +30,7 @@ import jakarta.persistence.Column; import jakarta.persistence.EmbeddedId; import jakarta.persistence.Entity; import jakarta.persistence.Table; +import jakarta.persistence.Transient; import java.io.Serializable; import static org.thingsboard.server.dao.model.ModelConstants.BOOLEAN_VALUE_COLUMN; @@ -65,19 +66,22 @@ public class AttributeKvEntity implements ToData, Serializable @Column(name = LAST_UPDATE_TS_COLUMN) private Long lastUpdateTs; + @Transient + protected String strKey; + @Override public AttributeKvEntry toData() { KvEntry kvEntry = null; if (strValue != null) { - kvEntry = new StringDataEntry(id.getAttributeKey(), strValue); + kvEntry = new StringDataEntry(strKey, strValue); } else if (booleanValue != null) { - kvEntry = new BooleanDataEntry(id.getAttributeKey(), booleanValue); + kvEntry = new BooleanDataEntry(strKey, booleanValue); } else if (doubleValue != null) { - kvEntry = new DoubleDataEntry(id.getAttributeKey(), doubleValue); + kvEntry = new DoubleDataEntry(strKey, doubleValue); } else if (longValue != null) { - kvEntry = new LongDataEntry(id.getAttributeKey(), longValue); + kvEntry = new LongDataEntry(strKey, longValue); } else if (jsonValue != null) { - kvEntry = new JsonDataEntry(id.getAttributeKey(), jsonValue); + kvEntry = new JsonDataEntry(strKey, jsonValue); } return new BaseAttributeKvEntry(kvEntry, lastUpdateTs); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java b/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java index d8c4ffddc9..5bf14d06ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.service; +import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; import org.thingsboard.common.util.RegexUtils; import org.thingsboard.server.common.data.id.EntityId; @@ -59,6 +60,19 @@ public class Validator { } } + /** + * This method validate String string. If string is null or can not be cast to enum than throw + * IncorrectParameterException exception + * + * @param val the val + * @param errorMessage the error message for exception + */ + public static > void validateEnum(Class enumClass, String val, String errorMessage) { + if (val == null || !EnumUtils.isValidEnum(enumClass, val)) { + throw new IncorrectParameterException(errorMessage); + } + } + /** * This method validate long value. If value isn't possitive than throw diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java new file mode 100644 index 0000000000..7af7df70ea --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.attributes; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.AttributeKvDictionary; +import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; + +import java.util.Optional; + +@SqlTsOrTsLatestAnyDao +public interface AttributeKvDictionaryRepository extends JpaRepository { + + Optional findByKeyId(int keyId); + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 75b7412985..6b33e45546 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -43,12 +43,12 @@ public abstract class AttributeKvInsertRepository { private static final String EMPTY_STR = ""; private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ? " + - "WHERE entity_type = ? and entity_id = ? and attribute_type =? and attribute_key = ?;"; + "WHERE entity_id = ? and attribute_type =? and attribute_key = ?;"; private static final String INSERT_OR_UPDATE = - "INSERT INTO attribute_kv (entity_type, entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts) " + - "VALUES(?, ?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?) " + - "ON CONFLICT (entity_type, entity_id, attribute_type, attribute_key) " + + "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?) " + + "ON CONFLICT (entity_id, attribute_type, attribute_key) " + "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?;"; @Autowired @@ -91,10 +91,9 @@ public abstract class AttributeKvInsertRepository { ps.setString(5, replaceNullChars(kvEntity.getJsonValue())); ps.setLong(6, kvEntity.getLastUpdateTs()); - ps.setString(7, kvEntity.getId().getEntityType().name()); - ps.setObject(8, kvEntity.getId().getEntityId()); - ps.setString(9, kvEntity.getId().getAttributeType()); - ps.setString(10, kvEntity.getId().getAttributeKey()); + ps.setObject(7, kvEntity.getId().getEntityId()); + ps.setInt(8, kvEntity.getId().getAttributeType()); + ps.setInt(9, kvEntity.getId().getAttributeKey()); } @Override @@ -121,43 +120,42 @@ public abstract class AttributeKvInsertRepository { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { AttributeKvEntity kvEntity = insertEntities.get(i); - ps.setString(1, kvEntity.getId().getEntityType().name()); - ps.setObject(2, kvEntity.getId().getEntityId()); - ps.setString(3, kvEntity.getId().getAttributeType()); - ps.setString(4, kvEntity.getId().getAttributeKey()); + ps.setObject(1, kvEntity.getId().getEntityId()); + ps.setInt(2, kvEntity.getId().getAttributeType()); + ps.setInt(3, kvEntity.getId().getAttributeKey()); - ps.setString(5, replaceNullChars(kvEntity.getStrValue())); - ps.setString(11, replaceNullChars(kvEntity.getStrValue())); + ps.setString(4, replaceNullChars(kvEntity.getStrValue())); + ps.setString(10, replaceNullChars(kvEntity.getStrValue())); if (kvEntity.getLongValue() != null) { - ps.setLong(6, kvEntity.getLongValue()); - ps.setLong(12, kvEntity.getLongValue()); + ps.setLong(5, kvEntity.getLongValue()); + ps.setLong(11, kvEntity.getLongValue()); } else { - ps.setNull(6, Types.BIGINT); - ps.setNull(12, Types.BIGINT); + ps.setNull(5, Types.BIGINT); + ps.setNull(11, Types.BIGINT); } if (kvEntity.getDoubleValue() != null) { - ps.setDouble(7, kvEntity.getDoubleValue()); - ps.setDouble(13, kvEntity.getDoubleValue()); + ps.setDouble(6, kvEntity.getDoubleValue()); + ps.setDouble(12, kvEntity.getDoubleValue()); } else { - ps.setNull(7, Types.DOUBLE); - ps.setNull(13, Types.DOUBLE); + ps.setNull(6, Types.DOUBLE); + ps.setNull(12, Types.DOUBLE); } if (kvEntity.getBooleanValue() != null) { - ps.setBoolean(8, kvEntity.getBooleanValue()); - ps.setBoolean(14, kvEntity.getBooleanValue()); + ps.setBoolean(7, kvEntity.getBooleanValue()); + ps.setBoolean(13, kvEntity.getBooleanValue()); } else { - ps.setNull(8, Types.BOOLEAN); - ps.setNull(14, Types.BOOLEAN); + ps.setNull(7, Types.BOOLEAN); + ps.setNull(13, Types.BOOLEAN); } - ps.setString(9, replaceNullChars(kvEntity.getJsonValue())); - ps.setString(15, replaceNullChars(kvEntity.getJsonValue())); + ps.setString(8, replaceNullChars(kvEntity.getJsonValue())); + ps.setString(14, replaceNullChars(kvEntity.getJsonValue())); - ps.setLong(10, kvEntity.getLastUpdateTs()); - ps.setLong(16, kvEntity.getLastUpdateTs()); + ps.setLong(9, kvEntity.getLastUpdateTs()); + ps.setLong(15, kvEntity.getLastUpdateTs()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index 6a0cdfabd0..a43be3e49d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -20,7 +20,6 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; @@ -29,35 +28,31 @@ import java.util.UUID; public interface AttributeKvRepository extends JpaRepository { - @Query("SELECT a FROM AttributeKvEntity a WHERE a.id.entityType = :entityType " + - "AND a.id.entityId = :entityId " + + @Query("SELECT a FROM AttributeKvEntity a WHERE a.id.entityId = :entityId " + "AND a.id.attributeType = :attributeType") - List findAllByEntityTypeAndEntityIdAndAttributeType(@Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("attributeType") String attributeType); + List findAllEntityIdAndAttributeType(@Param("entityId") UUID entityId, + @Param("attributeType") int attributeType); @Transactional @Modifying - @Query("DELETE FROM AttributeKvEntity a WHERE a.id.entityType = :entityType " + - "AND a.id.entityId = :entityId " + + @Query("DELETE FROM AttributeKvEntity a WHERE a.id.entityId = :entityId " + "AND a.id.attributeType = :attributeType " + "AND a.id.attributeKey = :attributeKey") - void delete(@Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("attributeType") String attributeType, - @Param("attributeKey") String attributeKey); + void delete(@Param("entityId") UUID entityId, + @Param("attributeType") int attributeType, + @Param("attributeKey") int attributeKey); @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = 'DEVICE' " + "AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId and device_profile_id = :deviceProfileId limit 100) ORDER BY attribute_key", nativeQuery = true) - List findAllKeysByDeviceProfileId(@Param("tenantId") UUID tenantId, + List findAllKeysByDeviceProfileId(@Param("tenantId") UUID tenantId, @Param("deviceProfileId") UUID deviceProfileId); @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = 'DEVICE' " + "AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId limit 100) ORDER BY attribute_key", nativeQuery = true) - List findAllKeysByTenantId(@Param("tenantId") UUID tenantId); + List findAllKeysByTenantId(@Param("tenantId") UUID tenantId); - @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = :entityType " + - "AND entity_id in :entityIds ORDER BY attribute_key", nativeQuery = true) - List findAllKeysByEntityIds(@Param("entityType") String entityType, @Param("entityIds") List entityIds); + @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE " + + "entity_id in :entityIds ORDER BY attribute_key", nativeQuery = true) + List findAllKeysByEntityIds(@Param("entityIds") List entityIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 6d195d1cc4..c47cd9dbdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -22,10 +22,12 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; +import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,6 +36,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; +import org.thingsboard.server.dao.model.sql.AttributeKvDictionary; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; @@ -46,6 +49,9 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.stream.Collectors; @@ -57,6 +63,9 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Autowired ScheduledLogExecutorComponent logExecutor; + @Autowired + private AttributeKvDictionaryRepository dictionaryRepository; + @Autowired private AttributeKvRepository attributeKvRepository; @@ -81,6 +90,9 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Value("${sql.batch_sort:true}") private boolean batchSortEnabled; + private final ConcurrentMap attributeDictionaryMap = new ConcurrentHashMap<>(); + private static final ReentrantLock attributeCreationLock = new ReentrantLock(); + private TbSqlBlockingQueueWrapper queue; @PostConstruct @@ -98,7 +110,6 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl queue = new TbSqlBlockingQueueWrapper<>(params, hashcodeFunction, batchThreads, statsFactory); queue.init(logExecutor, v -> attributeKvInsertRepository.saveOrUpdate(v), Comparator.comparing((AttributeKvEntity attributeKvEntity) -> attributeKvEntity.getId().getEntityId()) - .thenComparing(attributeKvEntity -> attributeKvEntity.getId().getEntityType().name()) .thenComparing(attributeKvEntity -> attributeKvEntity.getId().getAttributeType()) .thenComparing(attributeKvEntity -> attributeKvEntity.getId().getAttributeKey()) ); @@ -112,81 +123,131 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public Optional find(TenantId tenantId, EntityId entityId, String attributeType, String attributeKey) { + public Optional find(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, String attributeKey) { AttributeKvCompositeKey compositeKey = - getAttributeKvCompositeKey(entityId, attributeType, attributeKey); - return Optional.ofNullable(DaoUtil.getData(attributeKvRepository.findById(compositeKey))); + getAttributeKvCompositeKey(entityId, attributeScope.getId(), getOrSaveKeyId(attributeKey)); + Optional attributeKvEntityOptional = attributeKvRepository.findById(compositeKey); + if (attributeKvEntityOptional.isPresent()) { + AttributeKvEntity attributeKvEntity = attributeKvEntityOptional.get(); + attributeKvEntity.setStrKey(attributeKey); + return Optional.ofNullable(DaoUtil.getData(attributeKvEntity)); + } + return Optional.ofNullable(DaoUtil.getData(attributeKvEntityOptional)); } @Override - public List find(TenantId tenantId, EntityId entityId, String attributeType, Collection attributeKeys) { + public List find(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, Collection attributeKeys) { List compositeKeys = attributeKeys .stream() .map(attributeKey -> - getAttributeKvCompositeKey(entityId, attributeType, attributeKey)) + getAttributeKvCompositeKey(entityId, attributeScope.getId(), getOrSaveKeyId(attributeKey))) .collect(Collectors.toList()); - return DaoUtil.convertDataList(Lists.newArrayList(attributeKvRepository.findAllById(compositeKeys))); + List attributes = attributeKvRepository.findAllById(compositeKeys); + attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(getKey(attributeKvEntity.getId().getAttributeKey()))); + return DaoUtil.convertDataList(Lists.newArrayList(attributes)); } @Override - public List findAll(TenantId tenantId, EntityId entityId, String attributeType) { + public List findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope) { + List attributes = attributeKvRepository.findAllEntityIdAndAttributeType( + entityId.getId(), + attributeScope.getId()); + attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(getKey(attributeKvEntity.getId().getAttributeKey()))); return DaoUtil.convertDataList(Lists.newArrayList( - attributeKvRepository.findAllByEntityTypeAndEntityIdAndAttributeType( - entityId.getEntityType(), - entityId.getId(), - attributeType))); + attributes)); } @Override public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { if (deviceProfileId != null) { - return attributeKvRepository.findAllKeysByDeviceProfileId(tenantId.getId(), deviceProfileId.getId()); + return attributeKvRepository.findAllKeysByDeviceProfileId(tenantId.getId(), deviceProfileId.getId()) + .stream().map(this::getKey).collect(Collectors.toList()); } else { - return attributeKvRepository.findAllKeysByTenantId(tenantId.getId()); + return attributeKvRepository.findAllKeysByTenantId(tenantId.getId()) + .stream().map(this::getKey).collect(Collectors.toList()); } } @Override - public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { return attributeKvRepository - .findAllKeysByEntityIds(entityType.name(), entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + .findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())) + .stream().map(this::getKey).collect(Collectors.toList()); } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute) { AttributeKvEntity entity = new AttributeKvEntity(); - entity.setId(new AttributeKvCompositeKey(entityId.getEntityType(), entityId.getId(), attributeType, attribute.getKey())); + entity.setId(new AttributeKvCompositeKey(entityId.getId(), attributeScope.getId(), getOrSaveKeyId(attribute.getKey()))); entity.setLastUpdateTs(attribute.getLastUpdateTs()); entity.setStrValue(attribute.getStrValue().orElse(null)); entity.setDoubleValue(attribute.getDoubleValue().orElse(null)); entity.setLongValue(attribute.getLongValue().orElse(null)); entity.setBooleanValue(attribute.getBooleanValue().orElse(null)); entity.setJsonValue(attribute.getJsonValue().orElse(null)); - return addToQueue(entity); + return addToQueue(entity, attribute.getKey()); } - private ListenableFuture addToQueue(AttributeKvEntity entity) { - return Futures.transform(queue.add(entity), v -> entity.getId().getAttributeKey(), MoreExecutors.directExecutor()); + private ListenableFuture addToQueue(AttributeKvEntity entity, String key) { + return Futures.transform(queue.add(entity), v -> key, MoreExecutors.directExecutor()); } @Override - public List> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List keys) { + public List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys) { List> futuresList = new ArrayList<>(keys.size()); for (String key : keys) { futuresList.add(service.submit(() -> { - attributeKvRepository.delete(entityId.getEntityType(), entityId.getId(), attributeType, key); + attributeKvRepository.delete(entityId.getId(), attributeScope.getId(), getOrSaveKeyId(key)); return key; })); } return futuresList; } - private AttributeKvCompositeKey getAttributeKvCompositeKey(EntityId entityId, String attributeType, String attributeKey) { + private AttributeKvCompositeKey getAttributeKvCompositeKey(EntityId entityId, Integer attributeType, Integer attributeKey) { return new AttributeKvCompositeKey( - entityId.getEntityType(), entityId.getId(), attributeType, attributeKey); } + + private Integer getOrSaveKeyId(String attributeKey) { + Integer keyId = attributeDictionaryMap.get(attributeKey); + if (keyId == null) { + Optional byIdOptional = dictionaryRepository.findById(attributeKey); + if (byIdOptional.isEmpty()) { + attributeCreationLock.lock(); + try { + byIdOptional = dictionaryRepository.findById(attributeKey); + if (byIdOptional.isEmpty()) { + AttributeKvDictionary attributeKvDictionary = new AttributeKvDictionary(); + attributeKvDictionary.setKey(attributeKey); + try { + AttributeKvDictionary saved = dictionaryRepository.save(attributeKvDictionary); + attributeDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (DataIntegrityViolationException | ConstraintViolationException e) { + byIdOptional = dictionaryRepository.findById(attributeKey); + AttributeKvDictionary dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); + attributeDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = byIdOptional.get().getKeyId(); + } + } finally { + attributeCreationLock.unlock(); + } + } else { + keyId = byIdOptional.get().getKeyId(); + attributeDictionaryMap.put(attributeKey, keyId); + } + } + return keyId; + } + private String getKey(Integer attributeKey) { + Optional byKeyId = dictionaryRepository.findByKeyId(attributeKey); + return byKeyId.map(AttributeKvDictionary::getKey).orElse(null); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index e9adae29ff..200e93765d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -16,7 +16,7 @@ package org.thingsboard.server.dao.sql.query; import lombok.Data; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; @@ -262,22 +262,22 @@ public class EntityKeyMapping { String query; if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) { String join = (hasFilter() && hasFilterValues(ctx)) ? "inner join" : "left join"; - query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key=:%s_key_id ", - join, alias, alias, alias, entityTypeStr, alias, alias); - String scope; + query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.attribute_key=(select key_id from attribute_kv_dictionary where key = :%s_key_id) ", + join, alias, alias, alias, alias); + int scope; if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { - scope = DataConstants.CLIENT_SCOPE; + scope = AttributeScope.CLIENT_SCOPE.getId(); } else if (entityKey.getType().equals(EntityKeyType.SHARED_ATTRIBUTE)) { - scope = DataConstants.SHARED_SCOPE; + scope = AttributeScope.SHARED_SCOPE.getId();; } else { - scope = DataConstants.SERVER_SCOPE; + scope = AttributeScope.SERVER_SCOPE.getId();; } - query = String.format("%s AND %s.attribute_type='%s' %s", query, alias, scope, filterQuery); + query = String.format("%s AND %s.attribute_type=%s %s", query, alias, scope, filterQuery); } else { String join = (hasFilter() && hasFilterValues(ctx)) ? "join LATERAL" : "left join LATERAL"; - query = String.format("%s (select * from attribute_kv %s WHERE %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key=:%s_key_id %s " + + query = String.format("%s (select * from attribute_kv %s WHERE %s.entity_id=entities.id AND %s.attribute_key=(select key_id from attribute_kv_dictionary where key = :%s_key_id) %s " + "ORDER BY %s.last_update_ts DESC limit 1) as %s ON true", - join, alias, alias, alias, entityTypeStr, alias, alias, filterQuery, alias, alias); + join, alias, alias, alias, alias, filterQuery, alias, alias); } return query; } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 85b09e2053..f376a777d3 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -103,17 +103,22 @@ CREATE TABLE IF NOT EXISTS audit_log ( ) PARTITION BY RANGE (created_time); CREATE TABLE IF NOT EXISTS attribute_kv ( - entity_type varchar(255), entity_id uuid, - attribute_type varchar(255), - attribute_key varchar(255), + attribute_type int, + attribute_key int, bool_v boolean, str_v varchar(10000000), long_v bigint, dbl_v double precision, json_v json, last_update_ts bigint, - CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_type, entity_id, attribute_type, attribute_key) + CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) +); + +CREATE TABLE IF NOT EXISTS attribute_kv_dictionary ( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT attribute_key_id_pkey PRIMARY KEY (key) ); CREATE TABLE IF NOT EXISTS component_descriptor ( diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index ed15566268..0a60b64a50 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -104,7 +104,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -122,7 +122,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 8b2b80203e..3f8f380b03 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -272,7 +272,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -290,7 +290,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/dao/src/main/resources/sql/schema-views-and-functions.sql b/dao/src/main/resources/sql/schema-views-and-functions.sql index 1aa5647bde..4583b71e92 100644 --- a/dao/src/main/resources/sql/schema-views-and-functions.sql +++ b/dao/src/main/resources/sql/schema-views-and-functions.sql @@ -23,7 +23,7 @@ SELECT d.* , COALESCE(da.bool_v, FALSE) as active FROM device d LEFT JOIN customer c ON c.id = d.customer_id - LEFT JOIN attribute_kv da ON da.entity_type = 'DEVICE' AND da.entity_id = d.id AND da.attribute_type = 'SERVER_SCOPE' AND da.attribute_key = 'active'; + LEFT JOIN attribute_kv da ON da.entity_id = d.id AND da.attribute_type = 2 AND da.attribute_key = (select key_id from attribute_kv_dictionary where key = 'active'); DROP VIEW IF EXISTS device_info_active_ts_view CASCADE; CREATE OR REPLACE VIEW device_info_active_ts_view AS diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index ad23b7492b..1c798ea7de 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -29,6 +29,7 @@ import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -321,12 +322,13 @@ public class TbDeviceProfileNodeTest { device.setCustomerId(customerId); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "alarmEnabled" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); attributeKvEntity.setBooleanValue(Boolean.TRUE); + attributeKvEntity.setStrKey("alarmEnabled"); attributeKvEntity.setLastUpdateTs(System.currentTimeMillis()); AttributeKvEntry entry = attributeKvEntity.toData(); @@ -403,11 +405,12 @@ public class TbDeviceProfileNodeTest { device.setCustomerId(customerId); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, tenantId.getId(), "SERVER_SCOPE", "alarmEnabled" + tenantId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("alarmEnabled"); attributeKvEntity.setBooleanValue(Boolean.TRUE); attributeKvEntity.setLastUpdateTs(System.currentTimeMillis()); @@ -486,11 +489,12 @@ public class TbDeviceProfileNodeTest { DeviceProfileData deviceProfileData = new DeviceProfileData(); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "greaterAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("greaterAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); @@ -555,20 +559,22 @@ public class TbDeviceProfileNodeTest { DeviceProfileData deviceProfileData = new DeviceProfileData(); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "greaterAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("greaterAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); AttributeKvCompositeKey alarmDelayCompositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "alarm_delay" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 11 ); AttributeKvEntity alarmDelayAttributeKvEntity = new AttributeKvEntity(); alarmDelayAttributeKvEntity.setId(alarmDelayCompositeKey); + alarmDelayAttributeKvEntity.setStrKey("alarm_delay"); long alarmDelayInSeconds = 5L; alarmDelayAttributeKvEntity.setLongValue(alarmDelayInSeconds); alarmDelayAttributeKvEntity.setLastUpdateTs(0L); @@ -669,20 +675,22 @@ public class TbDeviceProfileNodeTest { AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "greaterAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("greaterAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); AttributeKvCompositeKey alarmDelayCompositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "alarm_delay" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 11 ); AttributeKvEntity alarmDelayAttributeKvEntity = new AttributeKvEntity(); alarmDelayAttributeKvEntity.setId(alarmDelayCompositeKey); + alarmDelayAttributeKvEntity.setStrKey("alarm_delay"); long alarmDelayInSeconds = 5L; alarmDelayAttributeKvEntity.setLongValue(alarmDelayInSeconds); alarmDelayAttributeKvEntity.setLastUpdateTs(0L); @@ -788,20 +796,22 @@ public class TbDeviceProfileNodeTest { DeviceProfileData deviceProfileData = new DeviceProfileData(); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "greaterAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("greaterAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); AttributeKvCompositeKey alarmDelayCompositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "alarm_delay" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 11 ); AttributeKvEntity alarmDelayAttributeKvEntity = new AttributeKvEntity(); alarmDelayAttributeKvEntity.setId(alarmDelayCompositeKey); + alarmDelayAttributeKvEntity.setStrKey("alarm_delay"); long alarmRepeating = 2; alarmDelayAttributeKvEntity.setLongValue(alarmRepeating); alarmDelayAttributeKvEntity.setLastUpdateTs(0L); @@ -891,7 +901,7 @@ public class TbDeviceProfileNodeTest { DeviceProfileData deviceProfileData = new DeviceProfileData(); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "greaterAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); Device device = new Device(); @@ -900,15 +910,17 @@ public class TbDeviceProfileNodeTest { AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("greaterAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); AttributeKvCompositeKey alarmDelayCompositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "alarm_delay" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 11 ); AttributeKvEntity alarmDelayAttributeKvEntity = new AttributeKvEntity(); alarmDelayAttributeKvEntity.setId(alarmDelayCompositeKey); + alarmDelayAttributeKvEntity.setStrKey("alarm_delay"); long repeatingCondition = 2; alarmDelayAttributeKvEntity.setLongValue(repeatingCondition); alarmDelayAttributeKvEntity.setLastUpdateTs(0L); @@ -1012,11 +1024,12 @@ public class TbDeviceProfileNodeTest { device.setCustomerId(customerId); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "greaterAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("greaterAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); @@ -1113,11 +1126,12 @@ public class TbDeviceProfileNodeTest { device.setCustomerId(customerId); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "greaterAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("greaterAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); @@ -1196,11 +1210,12 @@ public class TbDeviceProfileNodeTest { device.setCustomerId(customerId); AttributeKvCompositeKey compositeKeyActiveSchedule = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "dynamicValueActiveSchedule" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntityActiveSchedule = new AttributeKvEntity(); attributeKvEntityActiveSchedule.setId(compositeKeyActiveSchedule); + attributeKvEntityActiveSchedule.setStrKey("dynamicValueActiveSchedule"); attributeKvEntityActiveSchedule.setJsonValue( "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" ); @@ -1280,11 +1295,12 @@ public class TbDeviceProfileNodeTest { device.setCustomerId(customerId); AttributeKvCompositeKey compositeKeyInactiveSchedule = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "dynamicValueInactiveSchedule" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntityInactiveSchedule = new AttributeKvEntity(); attributeKvEntityInactiveSchedule.setId(compositeKeyInactiveSchedule); + attributeKvEntityInactiveSchedule.setStrKey("dynamicValueInactiveSchedule"); attributeKvEntityInactiveSchedule.setJsonValue( "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":false,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":6,\"startsOn\":0,\"endsOn\":0},{\"enabled\":false,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":0}],\"dynamicValue\":null}" ); @@ -1372,11 +1388,12 @@ public class TbDeviceProfileNodeTest { device.setCustomerId(customerId); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.CUSTOMER, deviceId.getId(), "SERVER_SCOPE", "lessAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("lessAttribute"); attributeKvEntity.setLongValue(30L); attributeKvEntity.setLastUpdateTs(0L); @@ -1447,11 +1464,12 @@ public class TbDeviceProfileNodeTest { DeviceProfileData deviceProfileData = new DeviceProfileData(); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "lessAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("lessAttribute"); attributeKvEntity.setLongValue(50L); attributeKvEntity.setLastUpdateTs(0L); @@ -1520,7 +1538,7 @@ public class TbDeviceProfileNodeTest { DeviceProfileData deviceProfileData = new DeviceProfileData(); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.TENANT, deviceId.getId(), "SERVER_SCOPE", "tenantAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); Device device = new Device(); @@ -1529,6 +1547,7 @@ public class TbDeviceProfileNodeTest { AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("tenantAttribute"); attributeKvEntity.setLongValue(100L); attributeKvEntity.setLastUpdateTs(0L); @@ -1605,7 +1624,7 @@ public class TbDeviceProfileNodeTest { DeviceProfileData deviceProfileData = new DeviceProfileData(); AttributeKvCompositeKey compositeKey = new AttributeKvCompositeKey( - EntityType.DEVICE, deviceId.getId(), EntityKeyType.SERVER_ATTRIBUTE.name(), "tenantAttribute" + deviceId.getId(), AttributeScope.SERVER_SCOPE.getId(), 10 ); Device device = new Device(); @@ -1614,6 +1633,7 @@ public class TbDeviceProfileNodeTest { AttributeKvEntity attributeKvEntity = new AttributeKvEntity(); attributeKvEntity.setId(compositeKey); + attributeKvEntity.setStrKey("tenantAttribute"); attributeKvEntity.setLongValue(100L); attributeKvEntity.setLastUpdateTs(0L); From e779c06ec1d39b27d6d3502aed468f5cf576aa9e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 18 Dec 2023 14:20:29 +0200 Subject: [PATCH 02/17] changed AttributeService interface to accept AttributeScope instead of String --- .../3.6.2/schema_update_attribute_kv.sql | 26 ------ .../data/upgrade/3.6.2/schema_update_ttl.sql | 87 ------------------- .../device/DeviceActorMessageProcessor.java | 19 ++-- .../controller/TelemetryController.java | 86 +++++++++--------- .../install/ThingsboardInstallService.java | 3 - .../device/ClaimDevicesServiceImpl.java | 5 +- .../device/DeviceProvisionServiceImpl.java | 5 +- .../service/edge/rpc/EdgeGrpcSession.java | 5 +- .../telemetry/BaseTelemetryProcessor.java | 3 +- .../rpc/sync/DefaultEdgeRequestsService.java | 3 +- .../DefaultTbEntityViewService.java | 25 +++--- .../CassandraTsDatabaseUpgradeService.java | 1 - .../DefaultSystemDataLoaderService.java | 7 +- .../install/SqlDatabaseUpgradeService.java | 11 +-- .../query/DefaultEntityQueryService.java | 14 +-- .../state/DefaultDeviceStateService.java | 5 +- .../DefaultSubscriptionManagerService.java | 3 +- .../subscription/TbAbstractSubCtx.java | 3 +- .../impl/DefaultEntityExportService.java | 10 +-- .../DefaultTelemetrySubscriptionService.java | 5 +- .../service/ws/DefaultWebSocketService.java | 10 +-- ...AbstractRuleEngineFlowIntegrationTest.java | 9 +- ...actRuleEngineLifecycleIntegrationTest.java | 3 +- .../dao/attributes/AttributesService.java | 13 +-- .../importing/csv/BulkImportColumnType.java | 6 +- .../dao/attributes/AttributeCacheKey.java | 3 +- .../server/dao/attributes/AttributeUtils.java | 4 +- .../dao/attributes/BaseAttributesService.java | 24 ++--- .../attributes/CachedAttributesService.java | 28 +++--- ...y.java => AttributeKvDictionaryEntry.java} | 2 +- .../server/dao/service/Validator.java | 14 --- .../AttributeKvDictionaryRepository.java | 8 +- .../dao/sql/attributes/JpaAttributeDao.java | 16 ++-- .../sql/schema-views-and-functions.sql | 72 +++++++++++++++ .../server/dao/service/EntityServiceTest.java | 17 ++-- .../attributes/BaseAttributesServiceTest.java | 40 ++++----- .../edge/BaseTbMsgPushNodeConfiguration.java | 4 +- .../engine/geo/TbGpsGeofencingActionNode.java | 5 +- .../rule/engine/math/TbMathNode.java | 13 +-- .../metadata/TbAbstractGetAttributesNode.java | 13 +-- .../metadata/TbAbstractGetMappedDataNode.java | 4 +- .../rule/engine/profile/DeviceState.java | 7 +- .../profile/DynamicPredicateValueCtxImpl.java | 3 +- .../engine/telemetry/TbMsgAttributesNode.java | 3 +- .../rule/engine/math/TbMathNodeTest.java | 3 +- .../metadata/TbGetAttributesNodeTest.java | 7 +- .../TbGetCustomerAttributeNodeTest.java | 5 +- .../TbGetRelatedAttributeNodeTest.java | 5 +- .../TbGetTenantAttributeNodeTest.java | 5 +- .../profile/TbDeviceProfileNodeTest.java | 54 ++++++------ 50 files changed, 343 insertions(+), 383 deletions(-) delete mode 100644 application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql rename dao/src/main/java/org/thingsboard/server/dao/model/sql/{AttributeKvDictionary.java => AttributeKvDictionaryEntry.java} (96%) diff --git a/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql b/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql index 142b3c1844..8a13029310 100644 --- a/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql +++ b/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql @@ -136,32 +136,6 @@ EXCEPTION END $$; -CREATE OR REPLACE PROCEDURE recreate_device_info_active_attribute_view() - LANGUAGE plpgsql AS -$$ -BEGIN - DROP VIEW IF EXISTS device_info_active_attribute_view CASCADE; - CREATE OR REPLACE VIEW device_info_active_attribute_view AS - SELECT d.* - , c.title as customer_title - , COALESCE((c.additional_info::json->>'isPublic')::bool, FALSE) as customer_is_public - , d.type as device_profile_name - , COALESCE(da.bool_v, FALSE) as active - FROM device d - LEFT JOIN customer c ON c.id = d.customer_id - LEFT JOIN attribute_kv da ON da.entity_id = d.id AND da.attribute_type = 2 AND da.attribute_key = (select key_id from attribute_kv_dictionary where key = 'active'); -END; -$$; - -CREATE OR REPLACE PROCEDURE recreate_device_info_view() - LANGUAGE plpgsql AS -$$ -BEGIN - DROP VIEW IF EXISTS device_info_view CASCADE; - CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view; -END; -$$; - CREATE OR REPLACE PROCEDURE drop_attribute_kv_old_table() LANGUAGE plpgsql AS $$ diff --git a/application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql b/application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql deleted file mode 100644 index 67ec25656e..0000000000 --- a/application/src/main/data/upgrade/3.6.2/schema_update_ttl.sql +++ /dev/null @@ -1,87 +0,0 @@ --- --- Copyright © 2016-2023 The Thingsboard Authors --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, - IN system_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE -tenant_cursor CURSOR FOR select tenant.id as tenant_id - from tenant; -tenant_id_record uuid; - customer_id_record uuid; - tenant_ttl bigint; - customer_ttl bigint; - deleted_for_entities bigint; - tenant_ttl_ts bigint; - customer_ttl_ts bigint; -BEGIN -OPEN tenant_cursor; -FETCH tenant_cursor INTO tenant_id_record; -WHILE FOUND - LOOP - EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', - tenant_id_record, 'TTL') INTO tenant_ttl; - if tenant_ttl IS NULL THEN - tenant_ttl := system_ttl; -END IF; - IF tenant_ttl > 0 THEN - tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; - deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; - deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; -END IF; -FOR customer_id_record IN -SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record - LOOP - EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', - customer_id_record, 'TTL') INTO customer_ttl; -IF customer_ttl IS NULL THEN - customer_ttl_ts := tenant_ttl_ts; -ELSE - IF customer_ttl > 0 THEN - customer_ttl_ts := - (EXTRACT(EPOCH FROM current_timestamp) * 1000 - - customer_ttl::bigint * 1000)::bigint; -END IF; -END IF; - IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN - deleted_for_entities := - delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; - deleted_for_entities := - delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; - deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, - customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; -END IF; -END LOOP; -FETCH tenant_cursor INTO tenant_id_record; -END LOOP; -END -$$; \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 99e61ee8dc..be7dad7abc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -32,6 +32,7 @@ import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EdgeUtils; @@ -501,7 +502,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso private void handleGetAttributesRequest(SessionInfoProto sessionInfo, GetAttributeRequestMsg request) { int requestId = request.getRequestId(); if (request.getOnlyShared()) { - Futures.addCallback(findAllAttributesByScope(DataConstants.SHARED_SCOPE), new FutureCallback<>() { + Futures.addCallback(findAllAttributesByScope(AttributeScope.SHARED_SCOPE), new FutureCallback<>() { @Override public void onSuccess(@Nullable List result) { GetAttributeResponseMsg responseMsg = GetAttributeResponseMsg.newBuilder() @@ -551,26 +552,26 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso ListenableFuture> clientAttributesFuture; ListenableFuture> sharedAttributesFuture; if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) { - clientAttributesFuture = findAllAttributesByScope(DataConstants.CLIENT_SCOPE); - sharedAttributesFuture = findAllAttributesByScope(DataConstants.SHARED_SCOPE); + clientAttributesFuture = findAllAttributesByScope(AttributeScope.CLIENT_SCOPE); + sharedAttributesFuture = findAllAttributesByScope(AttributeScope.SHARED_SCOPE); } else if (!CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) { - clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), DataConstants.CLIENT_SCOPE); - sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), DataConstants.SHARED_SCOPE); + clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), AttributeScope.CLIENT_SCOPE); + sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), AttributeScope.SHARED_SCOPE); } else if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) { clientAttributesFuture = Futures.immediateFuture(Collections.emptyList()); - sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), DataConstants.SHARED_SCOPE); + sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), AttributeScope.SHARED_SCOPE); } else { sharedAttributesFuture = Futures.immediateFuture(Collections.emptyList()); - clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), DataConstants.CLIENT_SCOPE); + clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), AttributeScope.CLIENT_SCOPE); } return Futures.allAsList(Arrays.asList(clientAttributesFuture, sharedAttributesFuture)); } - private ListenableFuture> findAllAttributesByScope(String scope) { + private ListenableFuture> findAllAttributesByScope(AttributeScope scope) { return systemContext.getAttributesService().findAll(tenantId, deviceId, scope); } - private ListenableFuture> findAttributesByScope(Set attributesSet, String scope) { + private ListenableFuture> findAttributesByScope(Set attributesSet, AttributeScope scope) { return systemContext.getAttributesService().find(tenantId, deviceId, scope, attributesSet); } diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 7056cab3f1..7033a9babc 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -51,7 +51,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.adaptor.JsonConverter; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; @@ -198,7 +198,7 @@ public class TelemetryController extends BaseController { public DeferredResult getAttributeKeysByScope( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) @PathVariable("scope") String scope) throws ThingsboardException { + @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) @PathVariable("scope") AttributeScope scope) throws ThingsboardException { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_ATTRIBUTES, entityType, entityIdStr, (result, tenantId, entityId) -> getAttributeKeysCallback(result, tenantId, entityId, scope)); } @@ -240,7 +240,7 @@ public class TelemetryController extends BaseController { public DeferredResult getAttributesByScope( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"}, required = true)) @PathVariable("scope") String scope, + @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"}, required = true)) @PathVariable("scope") AttributeScope scope, @Parameter(description = ATTRIBUTES_KEYS_DESCRIPTION) @RequestParam(name = "keys", required = false) String keysStr) throws ThingsboardException { SecurityUser user = getCurrentUser(); return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_ATTRIBUTES, entityType, entityIdStr, @@ -352,7 +352,7 @@ public class TelemetryController extends BaseController { @ResponseBody public DeferredResult saveDeviceAttributes( @Parameter(description = DEVICE_ID_PARAM_DESCRIPTION, required = true) @PathVariable("deviceId") String deviceIdStr, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE"}, required = true)) @PathVariable("scope") String scope, + @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE"}, required = true)) @PathVariable("scope") AttributeScope scope, @Parameter(description = ATTRIBUTES_JSON_REQUEST_DESCRIPTION, required = true) @RequestBody JsonNode request) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(EntityType.DEVICE, deviceIdStr); return saveAttributes(getTenantId(), entityId, scope, request); @@ -376,7 +376,7 @@ public class TelemetryController extends BaseController { public DeferredResult saveEntityAttributesV1( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE"})) @PathVariable("scope")String scope, + @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE"})) @PathVariable("scope")AttributeScope scope, @Parameter(description = ATTRIBUTES_JSON_REQUEST_DESCRIPTION, required = true)@RequestBody JsonNode request) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr); return saveAttributes(getTenantId(), entityId, scope, request); @@ -400,7 +400,7 @@ public class TelemetryController extends BaseController { public DeferredResult saveEntityAttributesV2( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE"}, required = true)) @PathVariable("scope")String scope, + @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE"}, required = true)) @PathVariable("scope")AttributeScope scope, @Parameter(description = ATTRIBUTES_JSON_REQUEST_DESCRIPTION, required = true)@RequestBody JsonNode request) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr); return saveAttributes(getTenantId(), entityId, scope, request); @@ -425,7 +425,7 @@ public class TelemetryController extends BaseController { public DeferredResult saveEntityTelemetry( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = TELEMETRY_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = "ANY")) @PathVariable("scope")String scope, + @Parameter(description = TELEMETRY_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = "ANY")) @PathVariable("scope")AttributeScope scope, @Parameter(description = TELEMETRY_JSON_REQUEST_DESCRIPTION, required = true)@RequestBody String requestBody) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr); return saveTelemetry(getTenantId(), entityId, requestBody, 0L); @@ -450,7 +450,7 @@ public class TelemetryController extends BaseController { public DeferredResult saveEntityTelemetryWithTTL( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = TELEMETRY_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = "ANY")) @PathVariable("scope")String scope, + @Parameter(description = TELEMETRY_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = "ANY")) @PathVariable("scope")AttributeScope scope, @Parameter(description = "A long value representing TTL (Time to Live) parameter.", required = true)@PathVariable("ttl")Long ttl, @Parameter(description = TELEMETRY_JSON_REQUEST_DESCRIPTION, required = true)@RequestBody String requestBody) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr); @@ -555,7 +555,7 @@ public class TelemetryController extends BaseController { @ResponseBody public DeferredResult deleteDeviceAttributes( @Parameter(description = DEVICE_ID_PARAM_DESCRIPTION, required = true) @PathVariable(DEVICE_ID) String deviceIdStr, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"}, required = true)) @PathVariable("scope")String scope, + @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"}, required = true)) @PathVariable("scope")AttributeScope scope, @Parameter(description = ATTRIBUTES_KEYS_DESCRIPTION, required = true)@RequestParam(name = "keys")String keysStr) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(EntityType.DEVICE, deviceIdStr); return deleteAttributes(entityId, scope, keysStr); @@ -579,49 +579,43 @@ public class TelemetryController extends BaseController { public DeferredResult deleteEntityAttributes( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) @PathVariable("scope")String scope, + @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) @PathVariable("scope")AttributeScope scope, @Parameter(description = ATTRIBUTES_KEYS_DESCRIPTION, required = true)@RequestParam(name = "keys")String keysStr) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr); return deleteAttributes(entityId, scope, keysStr); } - private DeferredResult deleteAttributes(EntityId entityIdSrc, String scope, String keysStr) throws ThingsboardException { + private DeferredResult deleteAttributes(EntityId entityIdSrc, AttributeScope scope, String keysStr) throws ThingsboardException { List keys = toKeysList(keysStr); if (keys.isEmpty()) { return getImmediateDeferredResult("Empty keys: " + keysStr, HttpStatus.BAD_REQUEST); } SecurityUser user = getCurrentUser(); - if (DataConstants.SERVER_SCOPE.equals(scope) || - DataConstants.SHARED_SCOPE.equals(scope) || - DataConstants.CLIENT_SCOPE.equals(scope)) { - return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { - tsSubService.deleteAndNotify(tenantId, entityId, scope, keys, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void tmp) { - logAttributesDeleted(user, entityId, scope, keys, null); - if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { - DeviceId deviceId = new DeviceId(entityId.getId()); - tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( - user.getTenantId(), deviceId, scope, keys), null); - } - result.setResult(new ResponseEntity<>(HttpStatus.OK)); + return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { + tsSubService.deleteAndNotify(tenantId, entityId, scope.name(), keys, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void tmp) { + logAttributesDeleted(user, entityId, scope, keys, null); + if (entityIdSrc.getEntityType().equals(EntityType.DEVICE)) { + DeviceId deviceId = new DeviceId(entityId.getId()); + tbClusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete( + user.getTenantId(), deviceId, scope.name(), keys), null); } + result.setResult(new ResponseEntity<>(HttpStatus.OK)); + } - @Override - public void onFailure(Throwable t) { - logAttributesDeleted(user, entityId, scope, keys, t); - result.setResult(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR)); - } - }); + @Override + public void onFailure(Throwable t) { + logAttributesDeleted(user, entityId, scope, keys, t); + result.setResult(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR)); + } }); - } else { - return getImmediateDeferredResult("Invalid attribute scope: " + scope, HttpStatus.BAD_REQUEST); - } + }); } - private DeferredResult saveAttributes(TenantId srcTenantId, EntityId entityIdSrc, String scope, JsonNode json) throws ThingsboardException { - if (!DataConstants.SERVER_SCOPE.equals(scope) && !DataConstants.SHARED_SCOPE.equals(scope)) { + private DeferredResult saveAttributes(TenantId srcTenantId, EntityId entityIdSrc, AttributeScope scope, JsonNode json) throws ThingsboardException { + if (AttributeScope.SERVER_SCOPE != scope && AttributeScope.SHARED_SCOPE != scope) { return getImmediateDeferredResult("Invalid scope: " + scope, HttpStatus.BAD_REQUEST); } if (json.isObject()) { @@ -636,7 +630,7 @@ public class TelemetryController extends BaseController { } SecurityUser user = getCurrentUser(); return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { - tsSubService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback() { + tsSubService.saveAndNotify(tenantId, entityId, scope.name(), attributes, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); @@ -710,10 +704,10 @@ public class TelemetryController extends BaseController { Futures.addCallback(future, getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); } - private void getAttributeValuesCallback(@Nullable DeferredResult result, SecurityUser user, EntityId entityId, String scope, String keys) { + private void getAttributeValuesCallback(@Nullable DeferredResult result, SecurityUser user, EntityId entityId, AttributeScope scope, String keys) { List keyList = toKeysList(keys); FutureCallback> callback = getAttributeValuesToResponseCallback(result, user, scope, entityId, keyList); - if (!StringUtils.isEmpty(scope)) { + if (scope != null) { if (keyList != null && !keyList.isEmpty()) { Futures.addCallback(attributesService.find(user.getTenantId(), entityId, scope, keyList), callback, MoreExecutors.directExecutor()); } else { @@ -721,7 +715,7 @@ public class TelemetryController extends BaseController { } } else { List>> futures = new ArrayList<>(); - for (String tmpScope : DataConstants.allScopes()) { + for (AttributeScope tmpScope : AttributeScope.values()) { if (keyList != null && !keyList.isEmpty()) { futures.add(attributesService.find(user.getTenantId(), entityId, tmpScope, keyList)); } else { @@ -735,13 +729,13 @@ public class TelemetryController extends BaseController { } } - private void getAttributeKeysCallback(@Nullable DeferredResult result, TenantId tenantId, EntityId entityId, String scope) { + private void getAttributeKeysCallback(@Nullable DeferredResult result, TenantId tenantId, EntityId entityId, AttributeScope scope) { Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), getAttributeKeysToResponseCallback(result), MoreExecutors.directExecutor()); } private void getAttributeKeysCallback(@Nullable DeferredResult result, TenantId tenantId, EntityId entityId) { List>> futures = new ArrayList<>(); - for (String scope : DataConstants.allScopes()) { + for (AttributeScope scope : AttributeScope.values()) { futures.add(attributesService.findAll(tenantId, entityId, scope)); } @@ -784,7 +778,7 @@ public class TelemetryController extends BaseController { } private FutureCallback> getAttributeValuesToResponseCallback(final DeferredResult response, - final SecurityUser user, final String scope, + final SecurityUser user, final AttributeScope scope, final EntityId entityId, final List keyList) { return new FutureCallback<>() { @Override @@ -835,18 +829,18 @@ public class TelemetryController extends BaseController { toException(e), telemetry); } - private void logAttributesDeleted(SecurityUser user, EntityId entityId, String scope, List keys, Throwable e) { + private void logAttributesDeleted(SecurityUser user, EntityId entityId, AttributeScope scope, List keys, Throwable e) { notificationEntityService.logEntityAction(user.getTenantId(), (UUIDBased & EntityId) entityId, ActionType.ATTRIBUTES_DELETED, user, toException(e), scope, keys); } - private void logAttributesUpdated(SecurityUser user, EntityId entityId, String scope, List attributes, Throwable e) { + private void logAttributesUpdated(SecurityUser user, EntityId entityId, AttributeScope scope, List attributes, Throwable e) { notificationEntityService.logEntityAction(user.getTenantId(), entityId, ActionType.ATTRIBUTES_UPDATED, user, toException(e), scope, attributes); } - private void logAttributesRead(SecurityUser user, EntityId entityId, String scope, List keys, Throwable e) { + private void logAttributesRead(SecurityUser user, EntityId entityId, AttributeScope scope, List keys, Throwable e) { notificationEntityService.logEntityAction(user.getTenantId(), entityId, ActionType.ATTRIBUTES_READ, user, toException(e), scope, keys); } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index ac348c1249..f9fcf77001 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -267,9 +267,6 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); case "3.6.2": log.info("Upgrading ThingsBoard from version 3.6.2 to 3.7.0 ..."); - if (databaseTsUpgradeService != null) { - databaseTsUpgradeService.upgradeDatabase("3.6.2"); - } databaseEntitiesUpgradeService.upgradeDatabase("3.6.2"); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; diff --git a/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java index a4ba2c8311..08852be7fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java @@ -29,6 +29,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -100,7 +101,7 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService { return Futures.immediateFailedFuture(new IllegalArgumentException()); } else { ListenableFuture> claimingAllowedFuture = attributesService.find(tenantId, device.getId(), - DataConstants.SERVER_SCOPE, Collections.singletonList(CLAIM_ATTRIBUTE_NAME)); + AttributeScope.SERVER_SCOPE, Collections.singletonList(CLAIM_ATTRIBUTE_NAME)); return Futures.transform(claimingAllowedFuture, list -> { if (list != null && !list.isEmpty()) { Optional claimingAllowedOptional = list.get(0).getBooleanValue(); @@ -123,7 +124,7 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService { return Futures.immediateFuture(new ClaimDataInfo(true, key, claimDataFromCache)); } else { ListenableFuture> claimDataAttrFuture = attributesService.find(device.getTenantId(), device.getId(), - DataConstants.SERVER_SCOPE, CLAIM_DATA_ATTRIBUTE_NAME); + AttributeScope.SERVER_SCOPE, CLAIM_DATA_ATTRIBUTE_NAME); return Futures.transform(claimDataAttrFuture, claimDataAttr -> { if (claimDataAttr.isPresent()) { diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index 763ce01116..c01c2330d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -189,7 +190,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { private ProvisionResponse processProvision(Device device, ProvisionRequest provisionRequest) { try { Optional provisionState = attributesService.find(device.getTenantId(), device.getId(), - DataConstants.SERVER_SCOPE, DEVICE_PROVISION_STATE).get(); + AttributeScope.SERVER_SCOPE, DEVICE_PROVISION_STATE).get(); if (provisionState != null && provisionState.isPresent() && !provisionState.get().getValueAsString().equals(PROVISIONED_STATE)) { notify(device, provisionRequest, TbMsgType.PROVISION_FAILURE, false); throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); @@ -245,7 +246,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { } private ListenableFuture> saveProvisionStateAttribute(Device device) { - return attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, + return attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), System.currentTimeMillis()))); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 59d744c3f1..003162816b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -25,6 +25,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.data.util.Pair; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; @@ -543,7 +544,7 @@ public final class EdgeGrpcSession implements Closeable { private ListenableFuture> getQueueStartTsAndSeqId() { ListenableFuture> future = - ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), DataConstants.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); + ctx.getAttributesService().find(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(QUEUE_START_TS_ATTR_KEY, QUEUE_START_SEQ_ID_ATTR_KEY)); return Futures.transform(future, attributeKvEntries -> { long startTs = 0L; long startSeqId = 0L; @@ -605,7 +606,7 @@ public final class EdgeGrpcSession implements Closeable { List attributes = Arrays.asList( new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_TS_ATTR_KEY, this.newStartTs), System.currentTimeMillis()), new BaseAttributeKvEntry(new LongDataEntry(QUEUE_START_SEQ_ID_ATTR_KEY, this.newStartSeqId), System.currentTimeMillis())); - return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), DataConstants.SERVER_SCOPE, attributes); + return ctx.getAttributesService().save(edge.getTenantId(), edge.getId(), AttributeScope.SERVER_SCOPE, attributes); } private DownlinkMsg convertEntityEventToDownlink(EdgeEvent edgeEvent) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 2f4c3ff2d0..6641753ae5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -30,6 +30,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -287,7 +288,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { String scope = attributeDeleteMsg.getScope(); List attributeKeys = attributeDeleteMsg.getAttributeNamesList(); - ListenableFuture> removeAllFuture = attributesService.removeAll(tenantId, entityId, scope, attributeKeys); + ListenableFuture> removeAllFuture = attributesService.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); return Futures.transformAsync(removeAllFuture, removeAttributes -> { if (EntityType.DEVICE.name().equals(entityType)) { SettableFuture futureToSet = SettableFuture.create(); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index 46bb940da9..71856ed61f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -28,6 +28,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -136,7 +137,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { return Futures.immediateFuture(null); } String scope = attributesRequestMsg.getScope(); - ListenableFuture> findAttrFuture = attributesService.findAll(tenantId, entityId, scope); + ListenableFuture> findAttrFuture = attributesService.findAll(tenantId, entityId, AttributeScope.valueOf(scope)); return Futures.transformAsync(findAttrFuture, ssAttributes -> processEntityAttributesAndAddToEdgeQueue(tenantId, entityId, edge, entityType, scope, ssAttributes, attributesRequestMsg), dbCallbackExecutorService); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index d511d6d4e4..665735cc22 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -24,6 +24,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.util.ConcurrentReferenceHashMap; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -99,9 +100,9 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen if (oldEntityView != null) { if (oldEntityView.getKeys() != null && oldEntityView.getKeys().getAttributes() != null) { - futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.CLIENT_SCOPE, oldEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SERVER_SCOPE, oldEntityView.getKeys().getAttributes().getSs(), user)); - futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SHARED_SCOPE, oldEntityView.getKeys().getAttributes().getSh(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, AttributeScope.CLIENT_SCOPE, oldEntityView.getKeys().getAttributes().getCs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, AttributeScope.SERVER_SCOPE, oldEntityView.getKeys().getAttributes().getSs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, AttributeScope.SHARED_SCOPE, oldEntityView.getKeys().getAttributes().getSh(), user)); } List tsKeys = oldEntityView.getKeys() != null && oldEntityView.getKeys().getTimeseries() != null ? oldEntityView.getKeys().getTimeseries() : Collections.emptyList(); @@ -109,9 +110,9 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen } if (savedEntityView.getKeys() != null) { if (savedEntityView.getKeys().getAttributes() != null) { - futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.SERVER_SCOPE, savedEntityView.getKeys().getAttributes().getSs(), user)); - futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.SHARED_SCOPE, savedEntityView.getKeys().getAttributes().getSh(), user)); + futures.add(copyAttributesFromEntityToEntityView(savedEntityView, AttributeScope.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), user)); + futures.add(copyAttributesFromEntityToEntityView(savedEntityView, AttributeScope.SERVER_SCOPE, savedEntityView.getKeys().getAttributes().getSs(), user)); + futures.add(copyAttributesFromEntityToEntityView(savedEntityView, AttributeScope.SHARED_SCOPE, savedEntityView.getKeys().getAttributes().getSh(), user)); } futures.add(copyLatestFromEntityToEntityView(tenantId, savedEntityView)); } @@ -269,7 +270,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen } } - private ListenableFuture> copyAttributesFromEntityToEntityView(EntityView entityView, String scope, Collection keys, User user) throws ThingsboardException { + private ListenableFuture> copyAttributesFromEntityToEntityView(EntityView entityView, AttributeScope scope, Collection keys, User user) throws ThingsboardException { EntityViewId entityId = entityView.getId(); if (keys != null && !keys.isEmpty()) { ListenableFuture> getAttrFuture = attributesService.find(entityView.getTenantId(), entityView.getEntityId(), scope, keys); @@ -287,7 +288,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen (startTime == 0 && endTime > lastUpdateTs) || (startTime < lastUpdateTs && endTime > lastUpdateTs); }).collect(Collectors.toList()); - tsSubService.saveAndNotify(entityView.getTenantId(), entityId, scope, attributes, new FutureCallback() { + tsSubService.saveAndNotify(entityView.getTenantId(), entityId, scope.name(), attributes, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { try { @@ -351,11 +352,11 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen }, MoreExecutors.directExecutor()); } - private ListenableFuture deleteAttributesFromEntityView(EntityView entityView, String scope, List keys, User user) { + private ListenableFuture deleteAttributesFromEntityView(EntityView entityView, AttributeScope scope, List keys, User user) { EntityViewId entityId = entityView.getId(); SettableFuture resultFuture = SettableFuture.create(); if (keys != null && !keys.isEmpty()) { - tsSubService.deleteAndNotify(entityView.getTenantId(), entityId, scope, keys, new FutureCallback() { + tsSubService.deleteAndNotify(entityView.getTenantId(), entityId, scope.name(), keys, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { try { @@ -433,11 +434,11 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen return resultFuture; } - private void logAttributesUpdated(TenantId tenantId, User user, EntityId entityId, String scope, List attributes, Throwable e) throws ThingsboardException { + private void logAttributesUpdated(TenantId tenantId, User user, EntityId entityId, AttributeScope scope, List attributes, Throwable e) throws ThingsboardException { notificationEntityService.logEntityAction(tenantId, entityId, ActionType.ATTRIBUTES_UPDATED, user, toException(e), scope, attributes); } - private void logAttributesDeleted(TenantId tenantId, User user, EntityId entityId, String scope, List keys, Throwable e) throws ThingsboardException { + private void logAttributesDeleted(TenantId tenantId, User user, EntityId entityId, AttributeScope scope, List keys, Throwable e) throws ThingsboardException { notificationEntityService.logEntityAction(tenantId, entityId, ActionType.ATTRIBUTES_DELETED, user, toException(e), scope, keys); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java index 3e2f29440c..5c7c51eb2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java @@ -52,7 +52,6 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase case "3.1.1": case "3.2.1": case "3.2.2": - case "3.6.2": break; default: throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 54c6dab5e5..402236c8c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -32,6 +32,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -469,7 +470,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { DeviceId t1Id = createDevice(demoTenant.getId(), null, savedThermostatDeviceProfile.getId(), "Thermostat T1", "T1_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); DeviceId t2Id = createDevice(demoTenant.getId(), null, savedThermostatDeviceProfile.getId(), "Thermostat T2", "T2_TEST_TOKEN", "Demo device for Thermostats dashboard").getId(); - attributesService.save(demoTenant.getId(), t1Id, DataConstants.SERVER_SCOPE, + attributesService.save(demoTenant.getId(), t1Id, AttributeScope.SERVER_SCOPE, Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.3948)), new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -122.1503)), new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("temperatureAlarmFlag", true)), @@ -477,7 +478,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("temperatureAlarmThreshold", (long) 20)), new BaseAttributeKvEntry(System.currentTimeMillis(), new LongDataEntry("humidityAlarmThreshold", (long) 50)))); - attributesService.save(demoTenant.getId(), t2Id, DataConstants.SERVER_SCOPE, + attributesService.save(demoTenant.getId(), t2Id, AttributeScope.SERVER_SCOPE, Arrays.asList(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("latitude", 37.493801)), new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("longitude", -121.948769)), new BaseAttributeKvEntry(System.currentTimeMillis(), new BooleanDataEntry("temperatureAlarmFlag", true)), @@ -586,7 +587,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } else { - ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, + ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) , System.currentTimeMillis()))); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 2983dea649..1a22b6e0e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -826,11 +826,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService executeQuery(conn, "CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);"); // remove temp files - if (pathToTempAttributeKvFile.toFile().exists()) { - boolean deleteTsKvFile = pathToTempAttributeKvFile.toFile().delete(); - if (deleteTsKvFile) { - log.info("Successfully deleted the temp file for attribute_kv table upgrade!"); - } + boolean deleteTsKvFile = Files.deleteIfExists(pathToTempAttributeKvFile); + if (deleteTsKvFile) { + log.info("Successfully deleted the temp file for attribute_kv table upgrade!"); } executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 3007000;"); @@ -900,9 +898,8 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService Statement statement = conn.createStatement(); statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script printWarnings(statement); - Thread.sleep(2000); log.info("Successfully executed query: {}", query); - } catch (InterruptedException | SQLException e) { + } catch (SQLException e) { log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); throw new RuntimeException("Failed to execute query:" + query + " due to: ", e); } diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index 296c992ccf..22912ebe9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -31,6 +31,7 @@ import org.springframework.util.CollectionUtils; import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -154,7 +155,7 @@ public class DefaultEntityQueryService implements EntityQueryService { try { Optional valueOpt = attributesService.find(user.getTenantId(), entityId, - TbAttributeSubscriptionScope.SERVER_SCOPE.name(), dynamicValue.getSourceAttribute()).get(); + AttributeScope.SERVER_SCOPE, dynamicValue.getSourceAttribute()).get(); if (valueOpt.isPresent()) { AttributeKvEntry entry = valueOpt.get(); @@ -251,13 +252,14 @@ public class DefaultEntityQueryService implements EntityQueryService { } if (isAttributes) { - ListenableFuture> future; - future = dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, ids)); - attributesKeysFuture = Futures.transform(future, list -> { - if (CollectionUtils.isEmpty(list)) { + Map> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType)); + List>> futures = new ArrayList<>(typesMap.size()); + typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, entityIds)))); + attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> { + if (CollectionUtils.isEmpty(lists)) { return Collections.emptyList(); } - return list.stream().distinct().sorted().collect(Collectors.toList()); + return lists.stream().flatMap(List::stream).distinct().sorted().collect(Collectors.toList()); }, dbCallbackExecutor); } else { attributesKeysFuture = null; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 64df0b0a43..0affb51fc4 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -36,6 +36,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.EntityType; @@ -585,7 +586,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES); future = Futures.transform(tsData, extractDeviceStateData(device), deviceStateExecutor); } else { - ListenableFuture> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), SERVER_SCOPE, PERSISTENT_ATTRIBUTES); + ListenableFuture> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, PERSISTENT_ATTRIBUTES); future = Futures.transform(attrData, extractDeviceStateData(device), deviceStateExecutor); } return transformInactivityTimeout(future); @@ -596,7 +597,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService { attributes.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> { if (inactivityTimeout > 0) { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index c3c050f73b..ed02fa4d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -23,6 +23,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.msg.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.AlarmInfo; @@ -496,7 +497,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene serviceId, subscription.getSessionId(), subscription.getSubscriptionId(), subscription.getEntityId()); final Map keyStates = subscription.getKeyStates(); - DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), DataConstants.CLIENT_SCOPE, keyStates.keySet()), values -> { + DonAsynchron.withCallback(attrService.find(subscription.getTenantId(), subscription.getEntityId(), AttributeScope.CLIENT_SCOPE, keyStates.keySet()), values -> { List missedUpdates = new ArrayList<>(); values.forEach(latestEntry -> { if (latestEntry.getLastUpdateTs() > keyStates.get(latestEntry.getKey())) { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractSubCtx.java index 7eba28bccd..b6046aef1d 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractSubCtx.java @@ -22,6 +22,7 @@ import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -215,7 +216,7 @@ public abstract class TbAbstractSubCtx { private ListenableFuture resolveEntityValue(TenantId tenantId, EntityId entityId, DynamicValueKey key) { ListenableFuture> entry = attributesService.find(tenantId, entityId, - TbAttributeSubscriptionScope.SERVER_SCOPE.name(), key.getSourceAttribute()); + AttributeScope.SERVER_SCOPE, key.getSourceAttribute()); return Futures.transform(entry, attributeOpt -> { DynamicValueKeySub sub = new DynamicValueKeySub(key, entityId); if (attributeOpt.isPresent()) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index a9e0cf1166..1e079e0c0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -19,7 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -108,16 +108,16 @@ public class DefaultEntityExportService> exportAttributes(EntitiesExportCtx ctx, E entity) throws ThingsboardException { - List scopes; + List scopes; if (entity.getId().getEntityType() == EntityType.DEVICE) { - scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); + scopes = List.of(AttributeScope.SERVER_SCOPE, AttributeScope.SHARED_SCOPE); } else { - scopes = Collections.singletonList(DataConstants.SERVER_SCOPE); + scopes = Collections.singletonList(AttributeScope.SERVER_SCOPE); } Map> attributes = new LinkedHashMap<>(); scopes.forEach(scope -> { try { - attributes.put(scope, attributesService.findAll(ctx.getTenantId(), entity.getId(), scope).get().stream() + attributes.put(scope.name(), attributesService.findAll(ctx.getTenantId(), entity.getId(), scope).get().stream() .map(attribute -> { AttributeExportData attributeExportData = new AttributeExportData(); attributeExportData.setKey(attribute.getKey()); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d65896a9d8..5139a6ee65 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -26,6 +26,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.id.CustomerId; @@ -248,7 +249,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { - ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); + ListenableFuture> saveFuture = attrService.save(tenantId, entityId, AttributeScope.valueOf(scope), attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } @@ -280,7 +281,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, FutureCallback callback) { - ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); + ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), keys); addVoidCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice)); } diff --git a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java index 711a218f0e..6a8d41208d 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java @@ -29,7 +29,7 @@ import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.CustomerId; @@ -949,7 +949,7 @@ public class DefaultWebSocketService implements WebSocketService { @Override public void onSuccess(@Nullable ValidationResult result) { List>> futures = new ArrayList<>(); - for (String scope : DataConstants.allScopes()) { + for (AttributeScope scope : AttributeScope.values()) { futures.add(attributesService.find(tenantId, entityId, scope, keys)); } @@ -968,7 +968,7 @@ public class DefaultWebSocketService implements WebSocketService { return new FutureCallback() { @Override public void onSuccess(@Nullable ValidationResult result) { - Futures.addCallback(attributesService.find(tenantId, entityId, scope, keys), callback, MoreExecutors.directExecutor()); + Futures.addCallback(attributesService.find(tenantId, entityId, AttributeScope.valueOf(scope), keys), callback, MoreExecutors.directExecutor()); } @Override @@ -983,7 +983,7 @@ public class DefaultWebSocketService implements WebSocketService { @Override public void onSuccess(@Nullable ValidationResult result) { List>> futures = new ArrayList<>(); - for (String scope : DataConstants.allScopes()) { + for (AttributeScope scope : AttributeScope.values()) { futures.add(attributesService.findAll(tenantId, entityId, scope)); } @@ -1002,7 +1002,7 @@ public class DefaultWebSocketService implements WebSocketService { return new FutureCallback() { @Override public void onSuccess(@Nullable ValidationResult result) { - Futures.addCallback(attributesService.findAll(tenantId, entityId, scope), callback, MoreExecutors.directExecutor()); + Futures.addCallback(attributesService.findAll(tenantId, entityId, AttributeScope.valueOf(scope)), callback, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index b86287c5d7..4cb36bb0a8 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -31,6 +31,7 @@ import org.thingsboard.rule.engine.util.TbMsgSource; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EventInfo; @@ -174,9 +175,9 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule device.setType("default"); device = doPost("/api/device", device, Device.class); - attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, + attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey1", "serverAttributeValue1"), System.currentTimeMillis()))).get(); - attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, + attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey2", "serverAttributeValue2"), System.currentTimeMillis()))).get(); TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); @@ -299,9 +300,9 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule device.setType("default"); device = doPost("/api/device", device, Device.class); - attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, + attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey1", "serverAttributeValue1"), System.currentTimeMillis()))).get(); - attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, + attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey2", "serverAttributeValue2"), System.currentTimeMillis()))).get(); TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 9b05be4a05..fa909b20e4 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -28,6 +28,7 @@ import org.thingsboard.rule.engine.util.TbMsgSource; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EventInfo; @@ -134,7 +135,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac device = doPost("/api/device", device, Device.class); log.warn("before update attr"); - attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, + attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey", "serverAttributeValue"), System.currentTimeMillis()))) .get(TIMEOUT, TimeUnit.SECONDS); log.warn("attr updated"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index de1386d0c6..ccbc90825a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -30,17 +31,17 @@ import java.util.Optional; */ public interface AttributesService { - ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey); + ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey); - ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys); + ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, Collection attributeKeys); - ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope); + ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope); - ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes); + ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); - ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); - ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys); + ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys); List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/importing/csv/BulkImportColumnType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/importing/csv/BulkImportColumnType.java index 87582b4d6d..4890d902b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/importing/csv/BulkImportColumnType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/importing/csv/BulkImportColumnType.java @@ -16,7 +16,7 @@ package org.thingsboard.server.common.data.sync.ie.importing.csv; import lombok.Getter; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode; @Getter @@ -24,8 +24,8 @@ public enum BulkImportColumnType { NAME, TYPE, LABEL, - SHARED_ATTRIBUTE(DataConstants.SHARED_SCOPE, true), - SERVER_ATTRIBUTE(DataConstants.SERVER_SCOPE, true), + SHARED_ATTRIBUTE(AttributeScope.SHARED_SCOPE.name(), true), + SERVER_ATTRIBUTE(AttributeScope.SERVER_SCOPE.name(), true), TIMESERIES(true), ACCESS_TOKEN, X509, diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java index f3fd154b0e..6fa1a7840f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.attributes; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; import java.io.Serializable; @@ -28,7 +29,7 @@ import java.io.Serializable; public class AttributeCacheKey implements Serializable { private static final long serialVersionUID = 2013369077925351881L; - private final String scope; + private final AttributeScope scope; private final EntityId entityId; private final String key; diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java index 29c44edd68..983661ff6e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java @@ -26,9 +26,9 @@ import java.util.List; public class AttributeUtils { - public static void validate(EntityId id, String scope) { + public static void validate(EntityId id, AttributeScope scope) { Validator.validateId(id.getId(), "Incorrect id " + id); - Validator.validateEnum(AttributeScope.class, scope, "Incorrect scope " + scope); + Validator.checkNotNull(scope, "Incorrect scope " + scope); } public static void validate(List kvEntries, boolean valueNoXssValidation) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 2defbd9398..3f8c88d609 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -54,23 +54,23 @@ public class BaseAttributesService implements AttributesService { } @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { + public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey); - return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey)); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKey)); } @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { + public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, Collection attributeKeys) { validate(entityId, scope); attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); - return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys)); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKeys)); } @Override - public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { + public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope) { validate(entityId, scope); - return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, AttributeScope.valueOf(scope))); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); } @Override @@ -84,23 +84,23 @@ public class BaseAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); + return attributesDao.save(tenantId, entityId, scope, attribute); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } @Override - public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys)); + return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 327f6b5706..cd633462a3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -105,7 +105,7 @@ public class CachedAttributesService implements AttributesService { @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { + public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey); @@ -120,7 +120,7 @@ public class CachedAttributesService implements AttributesService { return cacheExecutor.submit(() -> { var cacheTransaction = cache.newTransactionForKey(attributeCacheKey); try { - Optional result = attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey); + Optional result = attributesDao.find(tenantId, entityId, scope, attributeKey); cacheTransaction.putIfAbsent(attributeCacheKey, result.orElse(null)); cacheTransaction.commit(); return result; @@ -134,7 +134,7 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { + public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, Collection attributeKeys) { validate(entityId, scope); attributeKeys = new LinkedHashSet<>(attributeKeys); // deduplicate the attributes attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); @@ -159,7 +159,7 @@ public class CachedAttributesService implements AttributesService { var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); try { log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); - List result = attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), notFoundAttributeKeys); + List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); for (AttributeKvEntry foundInDbAttribute : result) { AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); @@ -181,7 +181,7 @@ public class CachedAttributesService implements AttributesService { }); } - private Map> findCachedAttributes(EntityId entityId, String scope, Collection attributeKeys) { + private Map> findCachedAttributes(EntityId entityId, AttributeScope scope, Collection attributeKeys) { Map> cachedAttributes = new HashMap<>(); for (String attributeKey : attributeKeys) { var cachedAttributeValue = cache.get(new AttributeCacheKey(scope, entityId, attributeKey)); @@ -196,9 +196,9 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { + public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope) { validate(entityId, scope); - return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, AttributeScope.valueOf(scope))); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); } @Override @@ -212,28 +212,28 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - ListenableFuture future = attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); return Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { - ListenableFuture future = attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); futures.add(Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor)); } return Futures.allAsList(futures); } - private String evict(EntityId entityId, String scope, AttributeKvEntry attribute, String key) { + private String evict(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute, String key) { log.trace("[{}][{}][{}] Before cache evict: {}", entityId, scope, key, attribute); cache.evictOrPut(new AttributeCacheKey(scope, entityId, key), attribute); log.trace("[{}][{}][{}] after cache evict.", entityId, scope, key); @@ -241,9 +241,9 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - List> futures = attributesDao.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); + List> futures = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, key -> { cache.evict(new AttributeCacheKey(scope, entityId, key)); return key; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionary.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java similarity index 96% rename from dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionary.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java index 1cde08912e..98e3a0f5ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionary.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java @@ -28,7 +28,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; @Data @Entity @Table(name = "attribute_kv_dictionary") -public final class AttributeKvDictionary { +public final class AttributeKvDictionaryEntry { @Id @Column(name = KEY_COLUMN) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java b/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java index 5bf14d06ae..d8c4ffddc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/Validator.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.service; -import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; import org.thingsboard.common.util.RegexUtils; import org.thingsboard.server.common.data.id.EntityId; @@ -60,19 +59,6 @@ public class Validator { } } - /** - * This method validate String string. If string is null or can not be cast to enum than throw - * IncorrectParameterException exception - * - * @param val the val - * @param errorMessage the error message for exception - */ - public static > void validateEnum(Class enumClass, String val, String errorMessage) { - if (val == null || !EnumUtils.isValidEnum(enumClass, val)) { - throw new IncorrectParameterException(errorMessage); - } - } - /** * This method validate long value. If value isn't possitive than throw diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java index 7af7df70ea..aa0ca5b8a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java @@ -16,14 +16,12 @@ package org.thingsboard.server.dao.sql.attributes; import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.dao.model.sql.AttributeKvDictionary; -import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; +import org.thingsboard.server.dao.model.sql.AttributeKvDictionaryEntry; import java.util.Optional; -@SqlTsOrTsLatestAnyDao -public interface AttributeKvDictionaryRepository extends JpaRepository { +public interface AttributeKvDictionaryRepository extends JpaRepository { - Optional findByKeyId(int keyId); + Optional findByKeyId(int keyId); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index c47cd9dbdf..50ee8c462c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; -import org.thingsboard.server.dao.model.sql.AttributeKvDictionary; +import org.thingsboard.server.dao.model.sql.AttributeKvDictionaryEntry; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; @@ -215,21 +215,21 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl private Integer getOrSaveKeyId(String attributeKey) { Integer keyId = attributeDictionaryMap.get(attributeKey); if (keyId == null) { - Optional byIdOptional = dictionaryRepository.findById(attributeKey); + Optional byIdOptional = dictionaryRepository.findById(attributeKey); if (byIdOptional.isEmpty()) { attributeCreationLock.lock(); try { byIdOptional = dictionaryRepository.findById(attributeKey); if (byIdOptional.isEmpty()) { - AttributeKvDictionary attributeKvDictionary = new AttributeKvDictionary(); - attributeKvDictionary.setKey(attributeKey); + AttributeKvDictionaryEntry attributeKvDictionaryEntry = new AttributeKvDictionaryEntry(); + attributeKvDictionaryEntry.setKey(attributeKey); try { - AttributeKvDictionary saved = dictionaryRepository.save(attributeKvDictionary); + AttributeKvDictionaryEntry saved = dictionaryRepository.save(attributeKvDictionaryEntry); attributeDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); } catch (DataIntegrityViolationException | ConstraintViolationException e) { byIdOptional = dictionaryRepository.findById(attributeKey); - AttributeKvDictionary dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); + AttributeKvDictionaryEntry dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); attributeDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); keyId = dictionary.getKeyId(); } @@ -247,7 +247,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return keyId; } private String getKey(Integer attributeKey) { - Optional byKeyId = dictionaryRepository.findByKeyId(attributeKey); - return byKeyId.map(AttributeKvDictionary::getKey).orElse(null); + Optional byKeyId = dictionaryRepository.findByKeyId(attributeKey); + return byKeyId.map(AttributeKvDictionaryEntry::getKey).orElse(null); } } diff --git a/dao/src/main/resources/sql/schema-views-and-functions.sql b/dao/src/main/resources/sql/schema-views-and-functions.sql index 4583b71e92..4d684c6406 100644 --- a/dao/src/main/resources/sql/schema-views-and-functions.sql +++ b/dao/src/main/resources/sql/schema-views-and-functions.sql @@ -291,3 +291,75 @@ CREATE OR REPLACE VIEW widget_type_info_view AS SELECT t.* , COALESCE((t.descriptor::json->>'type')::text, '') as widget_type FROM widget_type t; + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record uuid; + customer_id_record uuid; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN + OPEN tenant_cursor; + FETCH tenant_cursor INTO tenant_id_record; + WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; + END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; + END IF; + FOR customer_id_record IN + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + customer_id_record, 'TTL') INTO customer_ttl; + IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; + ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; + END IF; + END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + END IF; + END LOOP; + FETCH tenant_cursor INTO tenant_id_record; + END LOOP; +END +$$; \ No newline at end of file diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java index f41e92a305..b3b56ffcba 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java @@ -25,6 +25,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; @@ -373,7 +374,7 @@ public class EntityServiceTest extends AbstractServiceTest { List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); - attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); + attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); } Futures.allAsList(attributeFutures).get(); @@ -552,7 +553,7 @@ public class EntityServiceTest extends AbstractServiceTest { List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); - attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); + attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); } Futures.allAsList(attributeFutures).get(); @@ -627,7 +628,7 @@ public class EntityServiceTest extends AbstractServiceTest { List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { Asset asset = assets.get(i); - attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), DataConstants.SERVER_SCOPE)); + attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), AttributeScope.SERVER_SCOPE)); } Futures.allAsList(attributeFutures).get(); @@ -1436,7 +1437,7 @@ public class EntityServiceTest extends AbstractServiceTest { List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); - for (String currentScope : DataConstants.allScopes()) { + for (AttributeScope currentScope : AttributeScope.values()) { attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), currentScope)); } } @@ -1538,7 +1539,7 @@ public class EntityServiceTest extends AbstractServiceTest { List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); - attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); + attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); } Futures.allAsList(attributeFutures).get(); @@ -1816,7 +1817,7 @@ public class EntityServiceTest extends AbstractServiceTest { List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); - attributeFutures.add(saveStringAttribute(device.getId(), "attributeString", attributeStrings.get(i), DataConstants.CLIENT_SCOPE)); + attributeFutures.add(saveStringAttribute(device.getId(), "attributeString", attributeStrings.get(i), AttributeScope.CLIENT_SCOPE)); } Futures.allAsList(attributeFutures).get(); @@ -2149,13 +2150,13 @@ public class EntityServiceTest extends AbstractServiceTest { return filter; } - private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, String scope) { + private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, String scope) { + private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index a863310b06..5bf8d24117 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -26,7 +26,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.cache.TbTransactionalCache; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -71,8 +71,8 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { DeviceId deviceId = new DeviceId(Uuids.timeBased()); KvEntry attrValue = new StringDataEntry("attribute1", "value1"); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - attributesService.save(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, Collections.singletonList(attr)).get(); - Optional saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, attr.getKey()).get(); + attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attr)).get(); + Optional saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, attr.getKey()).get(); Assert.assertTrue(saved.isPresent()); Assert.assertEquals(attr, saved.get()); } @@ -83,17 +83,17 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { KvEntry attrOldValue = new StringDataEntry("attribute1", "value1"); AttributeKvEntry attrOld = new BaseAttributeKvEntry(attrOldValue, 42L); - attributesService.save(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, Collections.singletonList(attrOld)).get(); - Optional saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, attrOld.getKey()).get(); + attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attrOld)).get(); + Optional saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, attrOld.getKey()).get(); Assert.assertTrue(saved.isPresent()); Assert.assertEquals(attrOld, saved.get()); KvEntry attrNewValue = new StringDataEntry("attribute1", "value2"); AttributeKvEntry attrNew = new BaseAttributeKvEntry(attrNewValue, 73L); - attributesService.save(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, Collections.singletonList(attrNew)).get(); + attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attrNew)).get(); - saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, attrOld.getKey()).get(); + saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, attrOld.getKey()).get(); Assert.assertEquals(attrNew, saved.get()); } @@ -108,11 +108,11 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { KvEntry attrBNewValue = new StringDataEntry("B", "value3"); AttributeKvEntry attrBNew = new BaseAttributeKvEntry(attrBNewValue, 73L); - attributesService.save(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, Collections.singletonList(attrAOld)).get(); - attributesService.save(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, Collections.singletonList(attrANew)).get(); - attributesService.save(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE, Collections.singletonList(attrBNew)).get(); + attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attrAOld)).get(); + attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attrANew)).get(); + attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attrBNew)).get(); - List saved = attributesService.findAll(SYSTEM_TENANT_ID, deviceId, DataConstants.CLIENT_SCOPE).get(); + List saved = attributesService.findAll(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE).get(); Assert.assertNotNull(saved); Assert.assertEquals(2, saved.size()); @@ -123,7 +123,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { @Test public void testDummyRequestWithEmptyResult() throws Exception { - var future = attributesService.find(new TenantId(UUID.randomUUID()), new DeviceId(UUID.randomUUID()), DataConstants.SERVER_SCOPE, "TEST"); + var future = attributesService.find(new TenantId(UUID.randomUUID()), new DeviceId(UUID.randomUUID()), AttributeScope.SERVER_SCOPE, "TEST"); Assert.assertNotNull(future); var result = future.get(10, TimeUnit.SECONDS); Assert.assertTrue(result.isEmpty()); @@ -133,7 +133,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { public void testConcurrentTransaction() throws Exception { var tenantId = new TenantId(UUID.randomUUID()); var deviceId = new DeviceId(UUID.randomUUID()); - var scope = DataConstants.SERVER_SCOPE; + var scope = AttributeScope.SERVER_SCOPE; var key = "TEST"; var attrKey = new AttributeCacheKey(scope, deviceId, "TEST"); @@ -179,7 +179,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { public void testFetchAndUpdateEmpty() throws Exception { var tenantId = new TenantId(UUID.randomUUID()); var deviceId = new DeviceId(UUID.randomUUID()); - var scope = DataConstants.SERVER_SCOPE; + var scope = AttributeScope.SERVER_SCOPE; var key = "TEST"; Optional emptyValue = attributesService.find(tenantId, deviceId, scope, key).get(10, TimeUnit.SECONDS); @@ -193,7 +193,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { public void testFetchAndUpdateMulti() throws Exception { var tenantId = new TenantId(UUID.randomUUID()); var deviceId = new DeviceId(UUID.randomUUID()); - var scope = DataConstants.SERVER_SCOPE; + var scope = AttributeScope.SERVER_SCOPE; var key1 = "TEST1"; var key2 = "TEST2"; @@ -222,7 +222,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { } private void testConcurrentFetchAndUpdate(TenantId tenantId, DeviceId deviceId, ListeningExecutorService pool) throws Exception { - var scope = DataConstants.SERVER_SCOPE; + var scope = AttributeScope.SERVER_SCOPE; var key = "TEST"; saveAttribute(tenantId, deviceId, scope, key, OLD_VALUE); List> futures = new ArrayList<>(); @@ -236,7 +236,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { } private void testConcurrentFetchAndUpdateMulti(TenantId tenantId, DeviceId deviceId, ListeningExecutorService pool) throws Exception { - var scope = DataConstants.SERVER_SCOPE; + var scope = AttributeScope.SERVER_SCOPE; var key1 = "TEST1"; var key2 = "TEST2"; saveAttribute(tenantId, deviceId, scope, key1, OLD_VALUE); @@ -258,7 +258,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { Assert.assertEquals(NEW_VALUE, newResult.get(1)); } - private String getAttributeValue(TenantId tenantId, DeviceId deviceId, String scope, String key) { + private String getAttributeValue(TenantId tenantId, DeviceId deviceId, AttributeScope scope, String key) { try { Optional entry = attributesService.find(tenantId, deviceId, scope, key).get(10, TimeUnit.SECONDS); return entry.orElseThrow(RuntimeException::new).getStrValue().orElse("Unknown"); @@ -268,7 +268,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { } } - private List getAttributeValues(TenantId tenantId, DeviceId deviceId, String scope, List keys) { + private List getAttributeValues(TenantId tenantId, DeviceId deviceId, AttributeScope scope, List keys) { try { List entry = attributesService.find(tenantId, deviceId, scope, keys).get(10, TimeUnit.SECONDS); return entry.stream().map(e -> e.getStrValue().orElse(null)).collect(Collectors.toList()); @@ -278,7 +278,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { } } - private void saveAttribute(TenantId tenantId, DeviceId deviceId, String scope, String key, String s) { + private void saveAttribute(TenantId tenantId, DeviceId deviceId, AttributeScope scope, String key, String s) { try { AttributeKvEntry newEntry = new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, s)); attributesService.save(tenantId, deviceId, scope, Collections.singletonList(newEntry)).get(10, TimeUnit.SECONDS); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/BaseTbMsgPushNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/BaseTbMsgPushNodeConfiguration.java index 64f80a7b9a..ffa1d0ff06 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/BaseTbMsgPushNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/BaseTbMsgPushNodeConfiguration.java @@ -17,7 +17,7 @@ package org.thingsboard.rule.engine.edge; import lombok.Data; import org.thingsboard.rule.engine.api.NodeConfiguration; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; @Data public class BaseTbMsgPushNodeConfiguration implements NodeConfiguration { @@ -27,7 +27,7 @@ public class BaseTbMsgPushNodeConfiguration implements NodeConfiguration { try { Optional entry = ctx.getAttributesService() - .find(ctx.getTenantId(), msg.getOriginator(), DataConstants.SERVER_SCOPE, ctx.getServiceId()) + .find(ctx.getTenantId(), msg.getOriginator(), AttributeScope.SERVER_SCOPE, ctx.getServiceId()) .get(1, TimeUnit.MINUTES); if (entry.isPresent()) { JsonObject element = parser.parse(entry.get().getValueAsString()).getAsJsonObject(); @@ -120,7 +121,7 @@ public class TbGpsGeofencingActionNode extends AbstractGeofencingNode attributeKvEntryList = Collections.singletonList(entry); - ctx.getAttributesService().save(ctx.getTenantId(), entityId, DataConstants.SERVER_SCOPE, attributeKvEntryList); + ctx.getAttributesService().save(ctx.getTenantId(), entityId, AttributeScope.SERVER_SCOPE, attributeKvEntryList); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java index 095612b4e1..455e5c8da7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java @@ -33,6 +33,7 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; @@ -208,15 +209,15 @@ public class TbMathNode implements TbNode { } private ListenableFuture saveAttribute(TbContext ctx, TbMsg msg, double result, TbMathResult mathResultDef) { - String attributeScope = getAttributeScope(mathResultDef.getAttributeScope()); + AttributeScope attributeScope = getAttributeScope(mathResultDef.getAttributeScope()); if (isIntegerResult(mathResultDef, config.getOperation())) { var value = toIntValue(result); return ctx.getTelemetryService().saveAttrAndNotify( - ctx.getTenantId(), msg.getOriginator(), attributeScope, mathResultDef.getKey(), value); + ctx.getTenantId(), msg.getOriginator(), attributeScope.name(), mathResultDef.getKey(), value); } else { var value = toDoubleValue(mathResultDef, result); return ctx.getTelemetryService().saveAttrAndNotify( - ctx.getTenantId(), msg.getOriginator(), attributeScope, mathResultDef.getKey(), value); + ctx.getTenantId(), msg.getOriginator(), attributeScope.name(), mathResultDef.getKey(), value); } } @@ -384,7 +385,7 @@ public class TbMathNode implements TbNode { case MESSAGE_METADATA: return Futures.immediateFuture(TbMathArgumentValue.fromMessageMetadata(arg, argKey, msg.getMetaData())); case ATTRIBUTE: - String scope = getAttributeScope(arg.getAttributeScope()); + AttributeScope scope = getAttributeScope(arg.getAttributeScope()); return Futures.transform(ctx.getAttributesService().find(ctx.getTenantId(), msg.getOriginator(), scope, argKey), opt -> getTbMathArgumentValue(arg, opt, "Attribute: " + argKey + " with scope: " + scope + " not found for entity: " + msg.getOriginator()) , MoreExecutors.directExecutor()); @@ -402,8 +403,8 @@ public class TbMathNode implements TbNode { return CONSTANT.equals(type) ? keyPattern : TbNodeUtils.processPattern(keyPattern, msg); } - private String getAttributeScope(String attrScope) { - return StringUtils.isEmpty(attrScope) ? DataConstants.SERVER_SCOPE : attrScope; + private AttributeScope getAttributeScope(String attrScope) { + return StringUtils.isEmpty(attrScope) ? AttributeScope.SERVER_SCOPE : AttributeScope.valueOf(attrScope); } private TbMathArgumentValue getTbMathArgumentValue(TbMathArgument arg, Optional kvOpt, String error) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 50d71f88a0..bb17f61f75 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -29,6 +29,7 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.rule.engine.util.TbMsgSource; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -100,9 +101,9 @@ public abstract class TbAbstractGetAttributesNode>> failuresPairSet = ConcurrentHashMap.newKeySet(); var getKvEntryPairFutures = Futures.allAsList( getLatestTelemetry(ctx, entityId, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg), failuresPairSet), - getAttrAsync(ctx, entityId, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresPairSet), - getAttrAsync(ctx, entityId, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresPairSet), - getAttrAsync(ctx, entityId, SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg), failuresPairSet) + getAttrAsync(ctx, entityId, AttributeScope.CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg), failuresPairSet), + getAttrAsync(ctx, entityId, AttributeScope.SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg), failuresPairSet), + getAttrAsync(ctx, entityId, AttributeScope.SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg), failuresPairSet) ); withCallback(getKvEntryPairFutures, futuresList -> { var msgMetaData = msg.getMetaData().copy(); @@ -127,7 +128,7 @@ public abstract class TbAbstractGetAttributesNode>> getAttrAsync( TbContext ctx, EntityId entityId, - String scope, + AttributeScope scope, List keys, Set>> failuresPairSet ) { @@ -138,9 +139,9 @@ public abstract class TbAbstractGetAttributesNode { if (isTellFailureIfAbsent && attributeKvEntryList.size() != keys.size()) { List nonExistentKeys = getNonExistentKeys(attributeKvEntryList, keys); - failuresPairSet.add(new TbPair<>(scope, nonExistentKeys)); + failuresPairSet.add(new TbPair<>(scope.name(), nonExistentKeys)); } - return new TbPair<>(scope, attributeKvEntryList); + return new TbPair<>(scope.name(), attributeKvEntryList); }, ctx.getDbCallbackExecutor()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetMappedDataNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetMappedDataNode.java index 8362918e6d..1fd8832525 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetMappedDataNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetMappedDataNode.java @@ -25,6 +25,7 @@ import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.rule.engine.util.EntitiesFieldsAsyncLoader; import org.thingsboard.rule.engine.util.TbMsgSource; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.TbMsg; @@ -36,7 +37,6 @@ import java.util.Map; import java.util.stream.Collectors; import static org.thingsboard.common.util.DonAsynchron.withCallback; -import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @Slf4j public abstract class TbAbstractGetMappedDataNode extends TbAbstractNodeWithFetchTo { @@ -134,7 +134,7 @@ public abstract class TbAbstractGetMappedDataNode> getAttributesAsync(TbContext ctx, EntityId entityId, List attrKeys) { - var latest = ctx.getAttributesService().find(ctx.getTenantId(), entityId, SERVER_SCOPE, attrKeys); + var latest = ctx.getAttributesService().find(ctx.getTenantId(), entityId, AttributeScope.SERVER_SCOPE, attrKeys); return Futures.transform(latest, l -> l.stream() .map(i -> (KvEntry) i) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java index b38fd29af7..73a4cc1fcd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java @@ -21,6 +21,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.profile.state.PersistedAlarmState; import org.thingsboard.rule.engine.profile.state.PersistedDeviceState; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -382,9 +383,9 @@ class DeviceState { } } if (!attributeKeys.isEmpty()) { - addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.CLIENT_SCOPE, attributeKeys).get()); - addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SHARED_SCOPE, attributeKeys).get()); - addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, DataConstants.SERVER_SCOPE, attributeKeys).get()); + addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, AttributeScope.CLIENT_SCOPE, attributeKeys).get()); + addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, AttributeScope.SHARED_SCOPE, attributeKeys).get()); + addToSnapshot(result, ctx.getAttributesService().find(ctx.getTenantId(), originator, AttributeScope.SERVER_SCOPE, attributeKeys).get()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java index 93ee38992f..111861ad34 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DynamicPredicateValueCtxImpl.java @@ -17,6 +17,7 @@ package org.thingsboard.rule.engine.profile; import lombok.extern.slf4j.Slf4j; import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.CustomerId; @@ -62,7 +63,7 @@ public class DynamicPredicateValueCtxImpl implements DynamicPredicateValueCtx { private EntityKeyValue getValue(EntityId entityId, String key) { try { - Optional entry = ctx.getAttributesService().find(tenantId, entityId, DataConstants.SERVER_SCOPE, key).get(); + Optional entry = ctx.getAttributesService().find(tenantId, entityId, AttributeScope.SERVER_SCOPE, key).get(); if (entry.isPresent()) { return DeviceState.toEntityValue(entry.get()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 06498241fb..10c3a0c5db 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -28,6 +28,7 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; @@ -96,7 +97,7 @@ public class TbMsgAttributesNode implements TbNode { } List keys = newAttributes.stream().map(KvEntry::getKey).collect(Collectors.toList()); - ListenableFuture> findFuture = ctx.getAttributesService().find(ctx.getTenantId(), msg.getOriginator(), scope, keys); + ListenableFuture> findFuture = ctx.getAttributesService().find(ctx.getTenantId(), msg.getOriginator(), AttributeScope.valueOf(scope), keys); DonAsynchron.withCallback(findFuture, currentAttributes -> { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 9eec38cfae..8ee70f23d7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -39,6 +39,7 @@ import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -365,7 +366,7 @@ public class TbMathNodeTest { TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, originator, TbMsgMetaData.EMPTY, JacksonUtil.newObjectNode().toString()); - when(attributesService.find(tenantId, originator, DataConstants.SERVER_SCOPE, "a")) + when(attributesService.find(tenantId, originator, AttributeScope.SERVER_SCOPE, "a")) .thenReturn(Futures.immediateFuture(Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), new DoubleDataEntry("a", 2.0))))); when(tsService.findLatest(tenantId, originator, "b")) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java index 1b8c95ab22..ebbaf90bf6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.util.TbMsgSource; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -106,13 +107,13 @@ public class TbGetAttributesNodeTest { tsKeys = List.of("temperature", "humidity", "unknown"); ts = System.currentTimeMillis(); - when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, DataConstants.CLIENT_SCOPE, clientAttributes)) + when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.CLIENT_SCOPE, clientAttributes)) .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(clientAttributes, ts))); - when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, DataConstants.SERVER_SCOPE, serverAttributes)) + when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.SERVER_SCOPE, serverAttributes)) .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(serverAttributes, ts))); - when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, DataConstants.SHARED_SCOPE, sharedAttributes)) + when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.SHARED_SCOPE, sharedAttributes)) .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(sharedAttributes, ts))); when(timeseriesServiceMock.findLatest(TENANT_ID, ORIGINATOR, tsKeys)) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java index 291614464a..247d1d76bd 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetCustomerAttributeNodeTest.java @@ -33,6 +33,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.util.TbMsgSource; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -277,7 +278,7 @@ public class TbGetCustomerAttributeNodeTest { doReturn(device).when(deviceServiceMock).findDeviceById(eq(TENANT_ID), eq(device.getId())); when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); - when(attributesServiceMock.find(eq(TENANT_ID), eq(CUSTOMER_ID), eq(DataConstants.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) + when(attributesServiceMock.find(eq(TENANT_ID), eq(CUSTOMER_ID), eq(AttributeScope.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) .thenReturn(Futures.immediateFuture(attributesList)); when(ctxMock.getDbCallbackExecutor()).thenReturn(DB_EXECUTOR); @@ -324,7 +325,7 @@ public class TbGetCustomerAttributeNodeTest { doReturn(Futures.immediateFuture(user)).when(userServiceMock).findUserByIdAsync(eq(TENANT_ID), eq(user.getId())); when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); - when(attributesServiceMock.find(eq(TENANT_ID), eq(CUSTOMER_ID), eq(DataConstants.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) + when(attributesServiceMock.find(eq(TENANT_ID), eq(CUSTOMER_ID), eq(AttributeScope.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) .thenReturn(Futures.immediateFuture(attributesList)); when(ctxMock.getDbCallbackExecutor()).thenReturn(DB_EXECUTOR); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetRelatedAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetRelatedAttributeNodeTest.java index dcf5c22ea5..216c5cf1cb 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetRelatedAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetRelatedAttributeNodeTest.java @@ -34,6 +34,7 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.data.RelationsQuery; import org.thingsboard.rule.engine.util.TbMsgSource; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DataConstants; @@ -291,7 +292,7 @@ public class TbGetRelatedAttributeNodeTest { doReturn(Futures.immediateFuture(List.of(entityRelation))).when(relationServiceMock).findByQuery(eq(TENANT_ID), any()); when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); - when(attributesServiceMock.find(eq(TENANT_ID), eq(user.getId()), eq(DataConstants.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) + when(attributesServiceMock.find(eq(TENANT_ID), eq(user.getId()), eq(AttributeScope.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) .thenReturn(Futures.immediateFuture(attributes)); when(ctxMock.getDbCallbackExecutor()).thenReturn(DB_EXECUTOR); @@ -342,7 +343,7 @@ public class TbGetRelatedAttributeNodeTest { doReturn(Futures.immediateFuture(List.of(entityRelation))).when(relationServiceMock).findByQuery(eq(TENANT_ID), any()); when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); - when(attributesServiceMock.find(eq(TENANT_ID), eq(secondCustomer.getId()), eq(DataConstants.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) + when(attributesServiceMock.find(eq(TENANT_ID), eq(secondCustomer.getId()), eq(AttributeScope.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) .thenReturn(Futures.immediateFuture(attributes)); when(ctxMock.getDbCallbackExecutor()).thenReturn(DB_EXECUTOR); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTenantAttributeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTenantAttributeNodeTest.java index 87b34cffe0..8fb36f810b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTenantAttributeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTenantAttributeNodeTest.java @@ -32,6 +32,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.util.TbMsgSource; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -216,7 +217,7 @@ public class TbGetTenantAttributeNodeTest { when(ctxMock.getTenantId()).thenReturn(TENANT_ID); when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); - when(attributesServiceMock.find(eq(TENANT_ID), eq(TENANT_ID), eq(DataConstants.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) + when(attributesServiceMock.find(eq(TENANT_ID), eq(TENANT_ID), eq(AttributeScope.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) .thenReturn(Futures.immediateFuture(attributesList)); when(ctxMock.getDbCallbackExecutor()).thenReturn(DB_EXECUTOR); @@ -257,7 +258,7 @@ public class TbGetTenantAttributeNodeTest { when(ctxMock.getTenantId()).thenReturn(TENANT_ID); when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); - when(attributesServiceMock.find(eq(TENANT_ID), eq(TENANT_ID), eq(DataConstants.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) + when(attributesServiceMock.find(eq(TENANT_ID), eq(TENANT_ID), eq(AttributeScope.SERVER_SCOPE), argThat(new ListMatcher<>(expectedPatternProcessedKeysList)))) .thenReturn(Futures.immediateFuture(attributesList)); when(ctxMock.getDbCallbackExecutor()).thenReturn(DB_EXECUTOR); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 1c798ea7de..0b94f07255 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -374,7 +374,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(attrListListenableFuture); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -458,11 +458,11 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(Futures.immediateFuture(Collections.emptyList())); - Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.anyString(), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.any(AttributeScope.class), Mockito.anyString())) .thenReturn(Futures.immediateFuture(Optional.empty())); - Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.anyString(), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.any(AttributeScope.class), Mockito.anyString())) .thenReturn(attrListListenableFuture); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -532,7 +532,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFutureWithLess); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -628,7 +628,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFuture); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -748,13 +748,13 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.anyString(), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.any(AttributeScope.class), Mockito.anyString())) .thenReturn(optionalDurationAttribute); Mockito.when(ctx.getDeviceService().findDeviceById(tenantId, deviceId)) .thenReturn(device); - Mockito.when(attributesService.find(eq(tenantId), eq(customerId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(customerId), eq(AttributeScope.SERVER_SCOPE), Mockito.anyString())) .thenReturn(emptyOptional); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listNoDurationAttribute); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -865,7 +865,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFuture); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -977,13 +977,13 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.anyString(), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), Mockito.any(AttributeScope.class), Mockito.anyString())) .thenReturn(optionalDurationAttribute); Mockito.when(ctx.getDeviceService().findDeviceById(tenantId, deviceId)) .thenReturn(device); - Mockito.when(attributesService.find(eq(tenantId), eq(customerId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(customerId), eq(AttributeScope.SERVER_SCOPE), Mockito.anyString())) .thenReturn(emptyOptional); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listNoDurationAttribute); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1080,7 +1080,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFuture); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1179,7 +1179,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFuture); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1262,7 +1262,7 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFutureActiveSchedule); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1359,7 +1359,7 @@ public class TbDeviceProfileNodeTest { Mockito.when(alarmService.findLatestActiveByOriginatorAndType(tenantId, deviceId, "highTemperatureAlarm")) .thenReturn(null); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFutureInactiveSchedule); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1434,11 +1434,11 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFutureWithLess); Mockito.when(ctx.getDeviceService().findDeviceById(tenantId, deviceId)) .thenReturn(device); - Mockito.when(attributesService.find(eq(tenantId), eq(customerId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(customerId), eq(AttributeScope.SERVER_SCOPE), Mockito.anyString())) .thenReturn(optionalListenableFutureWithLess); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1510,9 +1510,9 @@ public class TbDeviceProfileNodeTest { .thenReturn(null); registerCreateAlarmMock(alarmService.createAlarm(any()), true); Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFutureWithLess); - Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(AttributeScope.SERVER_SCOPE), Mockito.anyString())) .thenReturn(optionalListenableFutureWithLess); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1592,11 +1592,11 @@ public class TbDeviceProfileNodeTest { Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); Mockito.when(ctx.getDeviceService().findDeviceById(tenantId, deviceId)) .thenReturn(device); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFutureWithLess); - Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.anyString(), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.any(AttributeScope.class), Mockito.anyString())) .thenReturn(emptyOptionalFuture); - Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(AttributeScope.SERVER_SCOPE), Mockito.anyString())) .thenReturn(optionalListenableFutureWithLess); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); @@ -1678,11 +1678,11 @@ public class TbDeviceProfileNodeTest { Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); Mockito.when(ctx.getDeviceService().findDeviceById(tenantId, deviceId)) .thenReturn(device); - Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.anyString(), Mockito.anySet())) + Mockito.when(attributesService.find(eq(tenantId), eq(deviceId), Mockito.any(AttributeScope.class), Mockito.anySet())) .thenReturn(listListenableFutureWithLess); - Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.anyString(), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(customerId), Mockito.any(AttributeScope.class), Mockito.anyString())) .thenReturn(emptyOptionalFuture); - Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(DataConstants.SERVER_SCOPE), Mockito.anyString())) + Mockito.when(attributesService.find(eq(tenantId), eq(tenantId), eq(AttributeScope.SERVER_SCOPE), Mockito.anyString())) .thenReturn(optionalListenableFutureWithLess); TbMsg theMsg = TbMsg.newMsg(TbMsgType.ALARM, deviceId, TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); From c7e06ec97e15b78d72a04b2a9d080b69c81afd79 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 20 Dec 2023 15:45:27 +0200 Subject: [PATCH 03/17] fixed TelemetryConrollerTest --- .../org/thingsboard/server/controller/TelemetryController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 7033a9babc..91e5010888 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -425,7 +425,7 @@ public class TelemetryController extends BaseController { public DeferredResult saveEntityTelemetry( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, - @Parameter(description = TELEMETRY_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = "ANY")) @PathVariable("scope")AttributeScope scope, + @Parameter(description = TELEMETRY_SCOPE_DESCRIPTION, required = true, schema = @Schema(allowableValues = "ANY")) @PathVariable("scope")String scope, @Parameter(description = TELEMETRY_JSON_REQUEST_DESCRIPTION, required = true)@RequestBody String requestBody) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndId(entityType, entityIdStr); return saveTelemetry(getTenantId(), entityId, requestBody, 0L); From c92a7ecf3a5d18b2f56c14a984ebe8e2bf7cf31e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 2 Jan 2024 17:46:29 +0200 Subject: [PATCH 04/17] deleted attribute_kv_dictionary table (used ts_kv_dictionary instead), renamed ts_kv_dictionary to key_dictionary --- .../schema_update_attribute_kv.sql | 27 +++++++----- .../install/ThingsboardInstallService.java | 6 +-- .../install/SqlDatabaseUpgradeService.java | 6 +-- .../CassandraTsLatestToSqlMigrateService.java | 24 +++++------ .../model/sql/AttributeKvDictionaryEntry.java | 41 ------------------- ...ey.java => KeyDictionaryCompositeKey.java} | 2 +- ...ictionary.java => KeyDictionaryEntry.java} | 6 +-- .../AttributeKvDictionaryRepository.java | 27 ------------ .../dao/sql/attributes/JpaAttributeDao.java | 22 +++++----- .../dao/sql/query/EntityKeyMapping.java | 6 +-- .../sqlts/BaseAbstractSqlTimeseriesDao.java | 24 +++++------ ...tory.java => KeyDictionaryRepository.java} | 11 +++-- .../latest/SearchTsKvLatestRepository.java | 4 +- .../sqlts/latest/TsKvLatestRepository.java | 18 ++++---- .../main/resources/sql/schema-entities.sql | 10 +---- .../main/resources/sql/schema-timescale.sql | 6 +-- dao/src/main/resources/sql/schema-ts-psql.sql | 12 +++--- .../sql/schema-views-and-functions.sql | 8 ++-- .../profile/TbDeviceProfileNodeTest.java | 3 -- .../tools/migrator/DictionaryParser.java | 2 +- 20 files changed, 97 insertions(+), 168 deletions(-) rename application/src/main/data/upgrade/{3.6.2 => 3.6.3}/schema_update_attribute_kv.sql (88%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/{TsKvDictionaryCompositeKey.java => KeyDictionaryCompositeKey.java} (93%) rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/{TsKvDictionary.java => KeyDictionaryEntry.java} (91%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java rename dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/{TsKvDictionaryRepository.java => KeyDictionaryRepository.java} (65%) diff --git a/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql b/application/src/main/data/upgrade/3.6.3/schema_update_attribute_kv.sql similarity index 88% rename from application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql rename to application/src/main/data/upgrade/3.6.3/schema_update_attribute_kv.sql index 8a13029310..509a94bf20 100644 --- a/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql +++ b/application/src/main/data/upgrade/3.6.3/schema_update_attribute_kv.sql @@ -46,13 +46,18 @@ $$ END; $$; --- create attribute_kv_dictionary table -CREATE TABLE IF NOT EXISTS attribute_kv_dictionary -( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT attribute_key_id_pkey PRIMARY KEY (key) -); +-- rename ts_kv_dictionary table to key_dictionary +DO +$$ + BEGIN + IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'ts_kv_dictionary') THEN + ALTER TABLE ts_kv_dictionary + RENAME CONSTRAINT ts_key_id_pkey TO key_id_pkey; + ALTER TABLE ts_kv_dictionary + RENAME TO key_dictionary; + END IF; + END; +$$; -- create to_attribute_type_id CREATE OR REPLACE FUNCTION to_attribute_type_id(IN attribute_type varchar, OUT attribute_type_id int) AS @@ -70,7 +75,7 @@ END; $$ LANGUAGE plpgsql; --- insert keys into attribute_kv_dictionary +-- insert keys into key_dictionary DO $$ DECLARE @@ -84,8 +89,8 @@ BEGIN LOOP FETCH key_cursor INTO insert_record; EXIT WHEN NOT FOUND; - IF NOT EXISTS(SELECT key FROM attribute_kv_dictionary WHERE key = insert_record.attribute_key) THEN - INSERT INTO attribute_kv_dictionary(key) VALUES (insert_record.attribute_key); + IF NOT EXISTS(SELECT key FROM key_dictionary WHERE key = insert_record.attribute_key) THEN + INSERT INTO key_dictionary(key) VALUES (insert_record.attribute_key); END IF; END LOOP; CLOSE key_cursor; @@ -122,7 +127,7 @@ BEGIN dbl_v, json_v, last_update_ts - FROM attribute_kv_old INNER JOIN attribute_kv_dictionary ON (attribute_kv_old.attribute_key = attribute_kv_dictionary.key) + FROM attribute_kv_old INNER JOIN key_dictionary ON (attribute_kv_old.attribute_key = key_dictionary.key) WHERE attribute_type= ANY(%L)) AS records) TO %L;', attribute_scope_array, path_to_file); EXECUTE format('COPY attribute_kv FROM %L', path_to_file); SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index f9fcf77001..383d22c21f 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -265,9 +265,9 @@ public class ThingsboardInstallService { case "3.6.0": log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); - case "3.6.2": - log.info("Upgrading ThingsBoard from version 3.6.2 to 3.7.0 ..."); - databaseEntitiesUpgradeService.upgradeDatabase("3.6.2"); + case "3.6.3": + log.info("Upgrading ThingsBoard from version 3.6.3 to 3.7.0 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.6.3"); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 1a22b6e0e2..1ad5c9afe4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -799,11 +799,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; - case "3.6.2": + case "3.6.3": try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - if (isOldSchema(conn, 3006002)) { + if (isOldSchema(conn, 3006003)) { log.info("Updating schema ..."); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.6.2", LOAD_ATTRIBUTE_KV_FUNCTIONS_SQL); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.6.3", LOAD_ATTRIBUTE_KV_FUNCTIONS_SQL); loadSql(schemaUpdateFile, conn); Path pathToTempAttributeKvFile; diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java index 4357e6be63..083d513e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java @@ -24,10 +24,10 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.dao.cassandra.CassandraCluster; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.util.NoSqlTsDao; import org.thingsboard.server.dao.util.SqlTsLatestDao; @@ -71,7 +71,7 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ protected CassandraCluster cluster; @Autowired - protected TsKvDictionaryRepository dictionaryRepository; + protected KeyDictionaryRepository dictionaryRepository; @Autowired private InstallScripts installScripts; @@ -192,22 +192,22 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ Integer keyId = tsKvDictionaryMap.get(strKey); if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (!tsKvDictionaryOptional.isPresent()) { tsCreationLock.lock(); try { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (!tsKvDictionaryOptional.isPresent()) { - TsKvDictionary tsKvDictionary = new TsKvDictionary(); - tsKvDictionary.setKey(strKey); + KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); + keyDictionaryEntry.setKey(strKey); try { - TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + KeyDictionaryEntry saved = dictionaryRepository.save(keyDictionaryEntry); tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); } catch (ConstraintViolationException e) { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); keyId = dictionary.getKeyId(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java deleted file mode 100644 index 98e3a0f5ab..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.model.sql; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import lombok.Data; -import org.hibernate.annotations.Generated; - -import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; - -@Data -@Entity -@Table(name = "attribute_kv_dictionary") -public final class AttributeKvDictionaryEntry { - - @Id - @Column(name = KEY_COLUMN) - private String key; - - @Column(name = KEY_ID_COLUMN, unique = true, columnDefinition="int") - @Generated - private int keyId; - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryCompositeKey.java similarity index 93% rename from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryCompositeKey.java index 49ab8b6822..599c0e5eb7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryCompositeKey.java @@ -25,7 +25,7 @@ import java.io.Serializable; @Data @NoArgsConstructor @AllArgsConstructor -public class TsKvDictionaryCompositeKey implements Serializable{ +public class KeyDictionaryCompositeKey implements Serializable{ @Transient private static final long serialVersionUID = -4089175869616037523L; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryEntry.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryEntry.java index 5f1ff6a9c1..6063cbf70a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryEntry.java @@ -28,9 +28,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; @Data @Entity -@Table(name = "ts_kv_dictionary") -@IdClass(TsKvDictionaryCompositeKey.class) -public final class TsKvDictionary { +@Table(name = "key_dictionary") +@IdClass(KeyDictionaryCompositeKey.class) +public final class KeyDictionaryEntry { @Id @Column(name = KEY_COLUMN) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java deleted file mode 100644 index aa0ca5b8a5..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.attributes; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.dao.model.sql.AttributeKvDictionaryEntry; - -import java.util.Optional; - -public interface AttributeKvDictionaryRepository extends JpaRepository { - - Optional findByKeyId(int keyId); - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 50ee8c462c..01149d3d49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -36,12 +36,14 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; -import org.thingsboard.server.dao.model.sql.AttributeKvDictionaryEntry; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; @@ -64,7 +66,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl ScheduledLogExecutorComponent logExecutor; @Autowired - private AttributeKvDictionaryRepository dictionaryRepository; + private KeyDictionaryRepository keyDictionaryRepository; @Autowired private AttributeKvRepository attributeKvRepository; @@ -215,21 +217,21 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl private Integer getOrSaveKeyId(String attributeKey) { Integer keyId = attributeDictionaryMap.get(attributeKey); if (keyId == null) { - Optional byIdOptional = dictionaryRepository.findById(attributeKey); + Optional byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); if (byIdOptional.isEmpty()) { attributeCreationLock.lock(); try { - byIdOptional = dictionaryRepository.findById(attributeKey); + byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); if (byIdOptional.isEmpty()) { - AttributeKvDictionaryEntry attributeKvDictionaryEntry = new AttributeKvDictionaryEntry(); + KeyDictionaryEntry attributeKvDictionaryEntry = new KeyDictionaryEntry(); attributeKvDictionaryEntry.setKey(attributeKey); try { - AttributeKvDictionaryEntry saved = dictionaryRepository.save(attributeKvDictionaryEntry); + KeyDictionaryEntry saved = keyDictionaryRepository.save(attributeKvDictionaryEntry); attributeDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); } catch (DataIntegrityViolationException | ConstraintViolationException e) { - byIdOptional = dictionaryRepository.findById(attributeKey); - AttributeKvDictionaryEntry dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); + byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); + KeyDictionaryEntry dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); attributeDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); keyId = dictionary.getKeyId(); } @@ -247,7 +249,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return keyId; } private String getKey(Integer attributeKey) { - Optional byKeyId = dictionaryRepository.findByKeyId(attributeKey); - return byKeyId.map(AttributeKvDictionaryEntry::getKey).orElse(null); + Optional byKeyId = keyDictionaryRepository.findByKeyId(attributeKey); + return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 200e93765d..23d8e972b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -256,13 +256,13 @@ public class EntityKeyMapping { } if (entityKey.getType().equals(EntityKeyType.TIME_SERIES)) { String join = (hasFilter() && hasFilterValues(ctx)) ? "inner join" : "left join"; - return String.format("%s ts_kv_latest %s ON %s.entity_id=entities.id AND %s.key = (select key_id from ts_kv_dictionary where key = :%s_key_id) %s", + return String.format("%s ts_kv_latest %s ON %s.entity_id=entities.id AND %s.key = (select key_id from key_dictionary where key = :%s_key_id) %s", join, alias, alias, alias, alias, filterQuery); } else { String query; if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) { String join = (hasFilter() && hasFilterValues(ctx)) ? "inner join" : "left join"; - query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.attribute_key=(select key_id from attribute_kv_dictionary where key = :%s_key_id) ", + query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.attribute_key=(select key_id from key_dictionary where key = :%s_key_id) ", join, alias, alias, alias, alias); int scope; if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { @@ -275,7 +275,7 @@ public class EntityKeyMapping { query = String.format("%s AND %s.attribute_type=%s %s", query, alias, scope, filterQuery); } else { String join = (hasFilter() && hasFilterValues(ctx)) ? "join LATERAL" : "left join LATERAL"; - query = String.format("%s (select * from attribute_kv %s WHERE %s.entity_id=entities.id AND %s.attribute_key=(select key_id from attribute_kv_dictionary where key = :%s_key_id) %s " + + query = String.format("%s (select * from attribute_kv %s WHERE %s.entity_id=entities.id AND %s.attribute_key=(select key_id from key_dictionary where key = :%s_key_id) %s " + "ORDER BY %s.last_update_ts DESC limit 1) as %s ON true", join, alias, alias, alias, alias, filterQuery, alias, alias); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 52c4216c69..9a33cba81e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -26,10 +26,10 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import jakarta.annotation.Nullable; import java.util.List; @@ -46,13 +46,13 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); protected static final ReentrantLock tsCreationLock = new ReentrantLock(); @Autowired - protected TsKvDictionaryRepository dictionaryRepository; + protected KeyDictionaryRepository dictionaryRepository; protected Integer getOrSaveKeyId(String strKey) { Integer keyId = tsKvDictionaryMap.get(strKey); if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (tsKvDictionaryOptional.isEmpty()) { tsCreationLock.lock(); try { @@ -60,17 +60,17 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni if (keyId != null) { return keyId; } - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (tsKvDictionaryOptional.isEmpty()) { - TsKvDictionary tsKvDictionary = new TsKvDictionary(); - tsKvDictionary.setKey(strKey); + KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); + keyDictionaryEntry.setKey(strKey); try { - TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + KeyDictionaryEntry saved = dictionaryRepository.save(keyDictionaryEntry); tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); } catch (DataIntegrityViolationException | ConstraintViolationException e) { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); keyId = dictionary.getKeyId(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java similarity index 65% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java index 41e165b92f..12c2422094 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java @@ -16,15 +16,14 @@ package org.thingsboard.server.dao.sqlts.dictionary; import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; -import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import java.util.Optional; -@SqlTsOrTsLatestAnyDao -public interface TsKvDictionaryRepository extends JpaRepository { +public interface KeyDictionaryRepository extends JpaRepository { + + Optional findByKeyId(int keyId); - Optional findByKeyId(int keyId); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java index beaa4a5f0d..fc7b2d568f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java @@ -30,9 +30,9 @@ public class SearchTsKvLatestRepository { public static final String FIND_ALL_BY_ENTITY_ID = "findAllByEntityId"; - public static final String FIND_ALL_BY_ENTITY_ID_QUERY = "SELECT ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key, ts_kv_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue," + + public static final String FIND_ALL_BY_ENTITY_ID_QUERY = "SELECT ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key, key_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue," + " ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue, ts_kv_latest.ts AS ts FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)"; + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)"; @PersistenceContext private EntityManager entityManager; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 835ce3a11b..abcae06e19 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -26,19 +26,19 @@ import java.util.UUID; public interface TsKvLatestRepository extends JpaRepository { - @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " + - "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE device_profile_id = :device_profile_id AND tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true) + @Query(value = "SELECT DISTINCT key_dictionary.key AS strKey FROM ts_kv_latest " + + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id " + + "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE device_profile_id = :device_profile_id AND tenant_id = :tenant_id limit 100) ORDER BY key_dictionary.key", nativeQuery = true) List getKeysByDeviceProfileId(@Param("tenant_id") UUID tenantId, @Param("device_profile_id") UUID deviceProfileId); - @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " + - "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true) + @Query(value = "SELECT DISTINCT key_dictionary.key AS strKey FROM ts_kv_latest " + + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id " + + "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE tenant_id = :tenant_id limit 100) ORDER BY key_dictionary.key", nativeQuery = true) List getKeysByTenantId(@Param("tenant_id") UUID tenantId); - @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " + - "WHERE ts_kv_latest.entity_id IN :entityIds ORDER BY ts_kv_dictionary.key", nativeQuery = true) + @Query(value = "SELECT DISTINCT key_dictionary.key AS strKey FROM ts_kv_latest " + + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id " + + "WHERE ts_kv_latest.entity_id IN :entityIds ORDER BY key_dictionary.key", nativeQuery = true) List findAllKeysByEntityIds(@Param("entityIds") List entityIds); } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index f376a777d3..9ce78bd56f 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -115,12 +115,6 @@ CREATE TABLE IF NOT EXISTS attribute_kv ( CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); -CREATE TABLE IF NOT EXISTS attribute_kv_dictionary ( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT attribute_key_id_pkey PRIMARY KEY (key) -); - CREATE TABLE IF NOT EXISTS component_descriptor ( id uuid NOT NULL CONSTRAINT component_descriptor_pkey PRIMARY KEY, created_time bigint NOT NULL, @@ -552,11 +546,11 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); -CREATE TABLE IF NOT EXISTS ts_kv_dictionary +CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, - CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) + CONSTRAINT key_id_pkey PRIMARY KEY (key) ); CREATE TABLE IF NOT EXISTS oauth2_params ( diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 0a60b64a50..f547a72cd1 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS ts_kv ( CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ); -CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( +CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) @@ -104,7 +104,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -122,7 +122,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 3f8f380b03..023e1b544d 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS ts_kv CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ) PARTITION BY RANGE (ts); -CREATE TABLE IF NOT EXISTS ts_kv_dictionary +CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, @@ -75,7 +75,7 @@ BEGIN WHERE schemaname = 'public' AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' + AND tablename != 'key_dictionary' AND tablename != 'ts_kv_indefinite' AND tablename != partition_by_max_ttl_date LOOP @@ -96,7 +96,7 @@ BEGIN WHERE schemaname = 'public' AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' + AND tablename != 'key_dictionary' AND tablename != 'ts_kv_indefinite' AND tablename != partition_by_max_ttl_date LOOP @@ -138,7 +138,7 @@ BEGIN WHERE schemaname = 'public' AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' + AND tablename != 'key_dictionary' AND tablename != 'ts_kv_indefinite' AND tablename != partition_by_max_ttl_date LOOP @@ -272,7 +272,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -290,7 +290,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/dao/src/main/resources/sql/schema-views-and-functions.sql b/dao/src/main/resources/sql/schema-views-and-functions.sql index 4d684c6406..bcb34a030a 100644 --- a/dao/src/main/resources/sql/schema-views-and-functions.sql +++ b/dao/src/main/resources/sql/schema-views-and-functions.sql @@ -23,7 +23,7 @@ SELECT d.* , COALESCE(da.bool_v, FALSE) as active FROM device d LEFT JOIN customer c ON c.id = d.customer_id - LEFT JOIN attribute_kv da ON da.entity_id = d.id AND da.attribute_type = 2 AND da.attribute_key = (select key_id from attribute_kv_dictionary where key = 'active'); + LEFT JOIN attribute_kv da ON da.entity_id = d.id AND da.attribute_type = 2 AND da.attribute_key = (select key_id from key_dictionary where key = 'active'); DROP VIEW IF EXISTS device_info_active_ts_view CASCADE; CREATE OR REPLACE VIEW device_info_active_ts_view AS @@ -34,7 +34,7 @@ SELECT d.* , COALESCE(dt.bool_v, FALSE) as active FROM device d LEFT JOIN customer c ON c.id = d.customer_id - LEFT JOIN ts_kv_latest dt ON dt.entity_id = d.id and dt.key = (select key_id from ts_kv_dictionary where key = 'active'); + LEFT JOIN ts_kv_latest dt ON dt.entity_id = d.id and dt.key = (select key_id from key_dictionary where key = 'active'); DROP VIEW IF EXISTS device_info_view CASCADE; CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view; @@ -312,7 +312,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -330,7 +330,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 0b94f07255..2aafcc0381 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -30,10 +30,8 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; import org.thingsboard.server.common.data.alarm.AlarmInfo; @@ -60,7 +58,6 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.DynamicValueSourceType; -import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.NumericFilterPredicate; diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java b/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java index 3b1164e95b..b85171d6ac 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java @@ -40,7 +40,7 @@ public class DictionaryParser { } private boolean isBlockStarted(String line) { - return line.startsWith("COPY public.ts_kv_dictionary ("); + return line.startsWith("COPY public.key_dictionary ("); } private void parseDictionaryDump(LineIterator iterator) throws IOException { From 2ad19106f369dbf6b2b2251c56a1569d5eec0e2c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 4 Jan 2024 14:52:12 +0200 Subject: [PATCH 05/17] removed redundant code, fixed tests --- .../service/install/TimescaleTsDatabaseUpgradeService.java | 5 ----- .../main/java/org/thingsboard/server/dao/JpaDaoConfig.java | 6 +++--- .../org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java | 1 - .../timeseries/nosql/TimeseriesServiceNoSqlTest.java | 1 - 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 4477bef1c4..205ec14591 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -184,11 +184,6 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr break; case "3.2.2": break; - case "3.6.2": - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - loadSql(conn, LOAD_TTL_FUNCTIONS_SQL, "3.6.2"); - } - break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java index 37ca4aecbc..40a45c3444 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java @@ -27,9 +27,9 @@ import org.thingsboard.server.dao.util.TbAutoConfiguration; */ @Configuration @TbAutoConfiguration -@ComponentScan({"org.thingsboard.server.dao.sql", "org.thingsboard.server.dao.sqlts", "org.thingsboard.server.dao.attributes", "org.thingsboard.server.dao.cache", "org.thingsboard.server.cache"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sql", "org.thingsboard.server.dao.sqlts"}) -@EntityScan({"org.thingsboard.server.dao.model.sql", "org.thingsboard.server.dao.model.sqlts"}) +@ComponentScan({"org.thingsboard.server.dao.sql", "org.thingsboard.server.dao.attributes", "org.thingsboard.server.dao.cache", "org.thingsboard.server.cache"}) +@EnableJpaRepositories("org.thingsboard.server.dao.sql") +@EntityScan("org.thingsboard.server.dao.model.sql") @EnableTransactionManagement public class JpaDaoConfig { diff --git a/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java index 44658f114f..9693e2d10b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.util.TbAutoConfiguration; @EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.dictionary"}) @EntityScan({"org.thingsboard.server.dao.model.sqlts.dictionary"}) @EnableTransactionManagement -@SqlTsOrTsLatestAnyDao public class SqlTimeseriesDaoConfig { } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java index af1b11174a..efcf62be20 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.service.timeseries.nosql; -import org.thingsboard.server.dao.AbstractJpaDaoTest; import org.thingsboard.server.dao.service.DaoNoSqlTest; import org.thingsboard.server.dao.service.timeseries.BaseTimeseriesServiceTest; From 237568997491b967e4b9b73be8e0c1ba9761b3ca Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 4 Jan 2024 18:55:00 +0200 Subject: [PATCH 06/17] refactoring: moved common method for retrieving key dictionary id to abstract class --- .../install/ThingsboardInstallService.java | 1 - .../install/SqlTsDatabaseUpgradeService.java | 2 - .../update/DefaultCacheCleanupService.java | 5 ++ ...paAbstractDaoListeningExecutorService.java | 55 +++++++++++++++++++ .../dao/sql/attributes/JpaAttributeDao.java | 47 ---------------- .../sqlts/BaseAbstractSqlTimeseriesDao.java | 54 ------------------ 6 files changed, 60 insertions(+), 104 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index ef5f3edd58..730ec0ec97 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -276,7 +276,6 @@ public class ThingsboardInstallService { } else { log.info("Skipping images migration. Run the upgrade with fromVersion as '3.6.2-images' to migrate"); } - //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache case "3.6.3": log.info("Upgrading ThingsBoard from version 3.6.3 to 3.7.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.3"); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java index b61da45631..6400463e11 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java @@ -213,8 +213,6 @@ public class SqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSer loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3"); } break; - case "3.6.2": - break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index 350005536a..e401ec2360 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE; import static org.thingsboard.server.common.data.CacheConstants.RESOURCE_INFO_CACHE; import static org.thingsboard.server.common.data.CacheConstants.SECURITY_SETTINGS_CACHE; @@ -94,6 +95,10 @@ public class DefaultCacheCleanupService implements CacheCleanupService { clearCacheByName(SECURITY_SETTINGS_CACHE); clearCacheByName(RESOURCE_INFO_CACHE); break; + case "3.6.3": + log.info("Clearing cache to upgrade from version 3.6.3 to 3.7.0"); + clearCacheByName(ATTRIBUTES_CACHE); + break; default: //Do nothing, since cache cleanup is optional. } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java index 5646aa1b6c..3a10b7f619 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java @@ -16,17 +16,29 @@ package org.thingsboard.server.dao.sql; import lombok.extern.slf4j.Slf4j; +import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.JdbcTemplate; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import javax.sql.DataSource; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; @Slf4j public abstract class JpaAbstractDaoListeningExecutorService { + private final ConcurrentMap keyDictionaryMap = new ConcurrentHashMap<>(); + protected static final ReentrantLock creationLock = new ReentrantLock(); + @Autowired protected JpaExecutorService service; @@ -36,6 +48,49 @@ public abstract class JpaAbstractDaoListeningExecutorService { @Autowired protected JdbcTemplate jdbcTemplate; + @Autowired + protected KeyDictionaryRepository keyDictionaryRepository; + + protected Integer getOrSaveKeyId(String strKey) { + Integer keyId = keyDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + if (tsKvDictionaryOptional.isEmpty()) { + creationLock.lock(); + try { + keyId = keyDictionaryMap.get(strKey); + if (keyId != null) { + return keyId; + } + tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + if (tsKvDictionaryOptional.isEmpty()) { + KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); + keyDictionaryEntry.setKey(strKey); + try { + KeyDictionaryEntry saved = keyDictionaryRepository.save(keyDictionaryEntry); + keyDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (DataIntegrityViolationException | ConstraintViolationException e) { + tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get KeyDictionaryEntry entity from DB!")); + keyDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + creationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + keyDictionaryMap.put(strKey, keyId); + } + } + return keyId; + } + protected void printWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 01149d3d49..54d0bf9495 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -22,10 +22,8 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; -import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -37,13 +35,11 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; -import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; @@ -51,9 +47,6 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.stream.Collectors; @@ -65,9 +58,6 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Autowired ScheduledLogExecutorComponent logExecutor; - @Autowired - private KeyDictionaryRepository keyDictionaryRepository; - @Autowired private AttributeKvRepository attributeKvRepository; @@ -92,9 +82,6 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Value("${sql.batch_sort:true}") private boolean batchSortEnabled; - private final ConcurrentMap attributeDictionaryMap = new ConcurrentHashMap<>(); - private static final ReentrantLock attributeCreationLock = new ReentrantLock(); - private TbSqlBlockingQueueWrapper queue; @PostConstruct @@ -214,40 +201,6 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl attributeKey); } - private Integer getOrSaveKeyId(String attributeKey) { - Integer keyId = attributeDictionaryMap.get(attributeKey); - if (keyId == null) { - Optional byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); - if (byIdOptional.isEmpty()) { - attributeCreationLock.lock(); - try { - byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); - if (byIdOptional.isEmpty()) { - KeyDictionaryEntry attributeKvDictionaryEntry = new KeyDictionaryEntry(); - attributeKvDictionaryEntry.setKey(attributeKey); - try { - KeyDictionaryEntry saved = keyDictionaryRepository.save(attributeKvDictionaryEntry); - attributeDictionaryMap.put(saved.getKey(), saved.getKeyId()); - keyId = saved.getKeyId(); - } catch (DataIntegrityViolationException | ConstraintViolationException e) { - byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); - KeyDictionaryEntry dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); - attributeDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); - keyId = dictionary.getKeyId(); - } - } else { - keyId = byIdOptional.get().getKeyId(); - } - } finally { - attributeCreationLock.unlock(); - } - } else { - keyId = byIdOptional.get().getKeyId(); - attributeDictionaryMap.put(attributeKey, keyId); - } - } - return keyId; - } private String getKey(Integer attributeKey) { Optional byKeyId = keyDictionaryRepository.findByKeyId(attributeKey); return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index e018cc1ef9..56f21aee39 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -19,75 +19,21 @@ import com.google.common.base.Function; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.hibernate.exception.ConstraintViolationException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DataIntegrityViolationException; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; -import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; -import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import jakarta.annotation.Nullable; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @Slf4j public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { - private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); - protected static final ReentrantLock tsCreationLock = new ReentrantLock(); - @Autowired - protected KeyDictionaryRepository keyDictionaryRepository; - - protected Integer getOrSaveKeyId(String strKey) { - Integer keyId = tsKvDictionaryMap.get(strKey); - if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); - if (tsKvDictionaryOptional.isEmpty()) { - tsCreationLock.lock(); - try { - keyId = tsKvDictionaryMap.get(strKey); - if (keyId != null) { - return keyId; - } - tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); - if (tsKvDictionaryOptional.isEmpty()) { - KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); - keyDictionaryEntry.setKey(strKey); - try { - KeyDictionaryEntry saved = keyDictionaryRepository.save(keyDictionaryEntry); - tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); - keyId = saved.getKeyId(); - } catch (DataIntegrityViolationException | ConstraintViolationException e) { - tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); - KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); - tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); - keyId = dictionary.getKeyId(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - } - } finally { - tsCreationLock.unlock(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - tsKvDictionaryMap.put(strKey, keyId); - } - } - return keyId; - } - protected ListenableFuture getReadTsKvQueryResultFuture(ReadTsKvQuery query, ListenableFuture>> future) { return Futures.transform(future, new Function<>() { @Nullable From 6b7ae41c72ce95dd9a4b37fd324a5d066e8e651e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 4 Jan 2024 19:23:43 +0200 Subject: [PATCH 07/17] refactoring: created KeyDictionaryDao --- .../dao/dictionary/KeyDictionaryDao.java | 25 +++++ ...paAbstractDaoListeningExecutorService.java | 55 ----------- .../dao/sql/attributes/JpaAttributeDao.java | 28 +++--- ...stractChunkedAggregationTimeseriesDao.java | 10 +- .../dao/sqlts/SqlTimeseriesLatestDao.java | 10 +- .../sqlts/dictionary/JpaKeyDictionaryDao.java | 92 +++++++++++++++++++ .../dao/sqlts/sql/JpaSqlTimeseriesDao.java | 5 +- .../timescale/TimescaleTimeseriesDao.java | 12 ++- 8 files changed, 156 insertions(+), 81 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java new file mode 100644 index 0000000000..aa663bf208 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.dictionary; + + +public interface KeyDictionaryDao { + + Integer getOrSaveKeyId(String strKey); + + String getKey(Integer attributeKey); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java index 3a10b7f619..5646aa1b6c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java @@ -16,29 +16,17 @@ package org.thingsboard.server.dao.sql; import lombok.extern.slf4j.Slf4j; -import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.JdbcTemplate; -import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; -import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; -import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import javax.sql.DataSource; import java.sql.SQLException; import java.sql.SQLWarning; import java.sql.Statement; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; @Slf4j public abstract class JpaAbstractDaoListeningExecutorService { - private final ConcurrentMap keyDictionaryMap = new ConcurrentHashMap<>(); - protected static final ReentrantLock creationLock = new ReentrantLock(); - @Autowired protected JpaExecutorService service; @@ -48,49 +36,6 @@ public abstract class JpaAbstractDaoListeningExecutorService { @Autowired protected JdbcTemplate jdbcTemplate; - @Autowired - protected KeyDictionaryRepository keyDictionaryRepository; - - protected Integer getOrSaveKeyId(String strKey) { - Integer keyId = keyDictionaryMap.get(strKey); - if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); - if (tsKvDictionaryOptional.isEmpty()) { - creationLock.lock(); - try { - keyId = keyDictionaryMap.get(strKey); - if (keyId != null) { - return keyId; - } - tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); - if (tsKvDictionaryOptional.isEmpty()) { - KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); - keyDictionaryEntry.setKey(strKey); - try { - KeyDictionaryEntry saved = keyDictionaryRepository.save(keyDictionaryEntry); - keyDictionaryMap.put(saved.getKey(), saved.getKeyId()); - keyId = saved.getKeyId(); - } catch (DataIntegrityViolationException | ConstraintViolationException e) { - tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); - KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get KeyDictionaryEntry entity from DB!")); - keyDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); - keyId = dictionary.getKeyId(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - } - } finally { - creationLock.unlock(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - keyDictionaryMap.put(strKey, keyId); - } - } - return keyId; - } - protected void printWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 54d0bf9495..7114d6db3e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -33,9 +33,9 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; @@ -67,6 +67,9 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Autowired private StatsFactory statsFactory; + @Autowired + private KeyDictionaryDao keyDictionaryDao; + @Value("${sql.attributes.batch_size:1000}") private int batchSize; @@ -114,7 +117,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Override public Optional find(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, String attributeKey) { AttributeKvCompositeKey compositeKey = - getAttributeKvCompositeKey(entityId, attributeScope.getId(), getOrSaveKeyId(attributeKey)); + getAttributeKvCompositeKey(entityId, attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(attributeKey)); Optional attributeKvEntityOptional = attributeKvRepository.findById(compositeKey); if (attributeKvEntityOptional.isPresent()) { AttributeKvEntity attributeKvEntity = attributeKvEntityOptional.get(); @@ -130,10 +133,10 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl attributeKeys .stream() .map(attributeKey -> - getAttributeKvCompositeKey(entityId, attributeScope.getId(), getOrSaveKeyId(attributeKey))) + getAttributeKvCompositeKey(entityId, attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(attributeKey))) .collect(Collectors.toList()); List attributes = attributeKvRepository.findAllById(compositeKeys); - attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(getKey(attributeKvEntity.getId().getAttributeKey()))); + attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(keyDictionaryDao.getKey(attributeKvEntity.getId().getAttributeKey()))); return DaoUtil.convertDataList(Lists.newArrayList(attributes)); } @@ -142,7 +145,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl List attributes = attributeKvRepository.findAllEntityIdAndAttributeType( entityId.getId(), attributeScope.getId()); - attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(getKey(attributeKvEntity.getId().getAttributeKey()))); + attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(keyDictionaryDao.getKey(attributeKvEntity.getId().getAttributeKey()))); return DaoUtil.convertDataList(Lists.newArrayList( attributes)); } @@ -151,10 +154,10 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { if (deviceProfileId != null) { return attributeKvRepository.findAllKeysByDeviceProfileId(tenantId.getId(), deviceProfileId.getId()) - .stream().map(this::getKey).collect(Collectors.toList()); + .stream().map(id -> keyDictionaryDao.getKey(id)).collect(Collectors.toList()); } else { return attributeKvRepository.findAllKeysByTenantId(tenantId.getId()) - .stream().map(this::getKey).collect(Collectors.toList()); + .stream().map(id -> keyDictionaryDao.getKey(id)).collect(Collectors.toList()); } } @@ -162,13 +165,13 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { return attributeKvRepository .findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())) - .stream().map(this::getKey).collect(Collectors.toList()); + .stream().map(id -> keyDictionaryDao.getKey(id)).collect(Collectors.toList()); } @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute) { AttributeKvEntity entity = new AttributeKvEntity(); - entity.setId(new AttributeKvCompositeKey(entityId.getId(), attributeScope.getId(), getOrSaveKeyId(attribute.getKey()))); + entity.setId(new AttributeKvCompositeKey(entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(attribute.getKey()))); entity.setLastUpdateTs(attribute.getLastUpdateTs()); entity.setStrValue(attribute.getStrValue().orElse(null)); entity.setDoubleValue(attribute.getDoubleValue().orElse(null)); @@ -187,7 +190,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl List> futuresList = new ArrayList<>(keys.size()); for (String key : keys) { futuresList.add(service.submit(() -> { - attributeKvRepository.delete(entityId.getId(), attributeScope.getId(), getOrSaveKeyId(key)); + attributeKvRepository.delete(entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(key)); return key; })); } @@ -200,9 +203,4 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl attributeType, attributeKey); } - - private String getKey(Integer attributeKey) { - Optional byKeyId = keyDictionaryRepository.findByKeyId(attributeKey); - return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 991043db88..c023b2ede3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; @@ -61,6 +62,9 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @Autowired private StatsFactory statsFactory; + @Autowired + private KeyDictionaryDao keyDictionaryDao; + @PostConstruct protected void init() { TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() @@ -93,7 +97,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return service.submit(() -> { tsKvRepository.delete( entityId.getId(), - getOrSaveKeyId(query.getKey()), + keyDictionaryDao.getOrSaveKeyId(query.getKey()), query.getStartTs(), query.getEndTs()); return null; @@ -132,7 +136,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } private ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { - Integer keyId = getOrSaveKeyId(query.getKey()); + Integer keyId = keyDictionaryDao.getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), keyId, @@ -160,7 +164,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } protected TsKvEntity switchAggregation(EntityId entityId, String key, long startTs, long endTs, Aggregation aggregation) { - var keyId = getOrSaveKeyId(key); + var keyId = keyDictionaryDao.getOrSaveKeyId(key); switch (aggregation) { case AVG: return tsKvRepository.findAvg(entityId.getId(), keyId, startTs, endTs); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 404855edf4..b7e871e3f5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; @@ -103,6 +104,9 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme @Autowired private StatsFactory statsFactory; + @Autowired + private KeyDictionaryDao keyDictionaryDao; + @PostConstruct protected void init() { TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() @@ -209,7 +213,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme TsKvLatestCompositeKey compositeKey = new TsKvLatestCompositeKey( entityId.getId(), - getOrSaveKeyId(key)); + keyDictionaryDao.getOrSaveKeyId(key)); Optional entry = tsKvLatestRepository.findById(compositeKey); if (entry.isPresent()) { TsKvLatestEntity tsKvLatestEntity = entry.get(); @@ -232,7 +236,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme if (ts >= query.getStartTs() && ts < query.getEndTs()) { TsKvLatestEntity latestEntity = new TsKvLatestEntity(); latestEntity.setEntityId(entityId.getId()); - latestEntity.setKey(getOrSaveKeyId(query.getKey())); + latestEntity.setKey(keyDictionaryDao.getOrSaveKeyId(query.getKey())); removedLatestFuture = service.submit(() -> { tsKvLatestRepository.delete(latestEntity); return true; @@ -259,7 +263,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme TsKvLatestEntity latestEntity = new TsKvLatestEntity(); latestEntity.setEntityId(entityId.getId()); latestEntity.setTs(tsKvEntry.getTs()); - latestEntity.setKey(getOrSaveKeyId(tsKvEntry.getKey())); + latestEntity.setKey(keyDictionaryDao.getOrSaveKeyId(tsKvEntry.getKey())); latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java new file mode 100644 index 0000000000..3c2a4cd228 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts.dictionary; + +import lombok.extern.slf4j.Slf4j; +import org.hibernate.exception.ConstraintViolationException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; + +@Component +@Slf4j +@SqlDao +public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService implements KeyDictionaryDao { + + private final ConcurrentMap keyDictionaryMap = new ConcurrentHashMap<>(); + protected static final ReentrantLock creationLock = new ReentrantLock(); + + @Autowired + private KeyDictionaryRepository keyDictionaryRepository; + + @Override + public Integer getOrSaveKeyId(String strKey) { + Integer keyId = keyDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + if (tsKvDictionaryOptional.isEmpty()) { + creationLock.lock(); + try { + keyId = keyDictionaryMap.get(strKey); + if (keyId != null) { + return keyId; + } + tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + if (tsKvDictionaryOptional.isEmpty()) { + KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); + keyDictionaryEntry.setKey(strKey); + try { + KeyDictionaryEntry saved = keyDictionaryRepository.save(keyDictionaryEntry); + keyDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (DataIntegrityViolationException | ConstraintViolationException e) { + tsKvDictionaryOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get KeyDictionaryEntry entity from DB!")); + keyDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + creationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + keyDictionaryMap.put(strKey, keyId); + } + } + return keyId; + } + + @Override + public String getKey(Integer attributeKey) { + Optional byKeyId = keyDictionaryRepository.findByKeyId(attributeKey); + return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java index 15a7e99129..4b0d4d9428 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sqlts.AbstractChunkedAggregationTimeseriesDao; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; @@ -59,6 +60,8 @@ public class JpaSqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao @Autowired private SqlPartitioningRepository partitioningRepository; + @Autowired + private KeyDictionaryDao keyDictionaryDao; private SqlTsPartitionDate tsFormat; @@ -83,7 +86,7 @@ public class JpaSqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); savePartitionIfNotExist(tsKvEntry.getTs()); String strKey = tsKvEntry.getKey(); - Integer keyId = getOrSaveKeyId(strKey); + Integer keyId = keyDictionaryDao.getOrSaveKeyId(strKey); TsKvEntity entity = new TsKvEntity(); entity.setEntityId(entityId.getId()); entity.setTs(tsKvEntry.getTs()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 128073323c..d7a6b45423 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; @@ -69,6 +70,9 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Autowired protected InsertTsRepository insertRepository; + @Autowired + protected KeyDictionaryDao keyDictionaryDao; + protected TbSqlBlockingQueueWrapper tsQueue; @PostConstruct @@ -108,7 +112,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { int dataPointDays = getDataPointDays(tsKvEntry, computeTtl(ttl)); String strKey = tsKvEntry.getKey(); - Integer keyId = getOrSaveKeyId(strKey); + Integer keyId = keyDictionaryDao.getOrSaveKeyId(strKey); TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); entity.setEntityId(entityId.getId()); entity.setTs(tsKvEntry.getTs()); @@ -130,7 +134,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Override public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { String strKey = query.getKey(); - Integer keyId = getOrSaveKeyId(strKey); + Integer keyId = keyDictionaryDao.getOrSaveKeyId(strKey); return service.submit(() -> { tsKvRepository.delete( entityId.getId(), @@ -161,7 +165,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements private ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); - Integer keyId = getOrSaveKeyId(strKey); + Integer keyId = keyDictionaryDao.getOrSaveKeyId(strKey); List timescaleTsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), keyId, @@ -205,7 +209,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } private List switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId) { - Integer keyId = getOrSaveKeyId(key); + Integer keyId = keyDictionaryDao.getOrSaveKeyId(key); switch (aggregation) { case AVG: return aggregationRepository.findAvg(entityId, keyId, timeBucket, startTs, endTs); From 4c77a75ef63df33768dbb10ac4c38d3750ea05ca Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 5 Jan 2024 10:49:33 +0200 Subject: [PATCH 08/17] fixed tests --- .../java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java | 2 ++ .../org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java index 9693e2d10b..9b3818794e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao; import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; @@ -24,6 +25,7 @@ import org.thingsboard.server.dao.util.TbAutoConfiguration; @Configuration @TbAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.dictionary"}) @EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.dictionary"}) @EntityScan({"org.thingsboard.server.dao.model.sqlts.dictionary"}) @EnableTransactionManagement diff --git a/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java index aa663bf208..64da0dd55d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java @@ -20,6 +20,6 @@ public interface KeyDictionaryDao { Integer getOrSaveKeyId(String strKey); - String getKey(Integer attributeKey); + String getKey(Integer keyId); } From a89985efdd4bad3ed6b81cf8113f74dda5974a90 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 5 Jan 2024 14:20:00 +0200 Subject: [PATCH 09/17] fixed AttributeKvRepository --- .../server/dao/sql/attributes/AttributeKvRepository.java | 4 ++-- .../server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index a43be3e49d..f5487971c9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -42,12 +42,12 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByDeviceProfileId(@Param("tenantId") UUID tenantId, @Param("deviceProfileId") UUID deviceProfileId); - @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE entity_type = 'DEVICE' " + + @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE " + "AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId limit 100) ORDER BY attribute_key", nativeQuery = true) List findAllKeysByTenantId(@Param("tenantId") UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java index 3c2a4cd228..90bb3bbed0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -84,8 +84,8 @@ public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService } @Override - public String getKey(Integer attributeKey) { - Optional byKeyId = keyDictionaryRepository.findByKeyId(attributeKey); + public String getKey(Integer keyId) { + Optional byKeyId = keyDictionaryRepository.findByKeyId(keyId); return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); } From 9619fb7fc5df1ec4cc0f348b2993cf43905d3c70 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 5 Jan 2024 14:23:12 +0200 Subject: [PATCH 10/17] fixed AttributeKvRepository --- .../server/dao/sql/attributes/AttributeKvRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index f5487971c9..064fa1fdc7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -43,12 +43,12 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByDeviceProfileId(@Param("tenantId") UUID tenantId, @Param("deviceProfileId") UUID deviceProfileId); @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE " + - "AND entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId limit 100) ORDER BY attribute_key", nativeQuery = true) + "entity_id in (SELECT id FROM device WHERE tenant_id = :tenantId limit 100) ORDER BY attribute_key", nativeQuery = true) List findAllKeysByTenantId(@Param("tenantId") UUID tenantId); @Query(value = "SELECT DISTINCT attribute_key FROM attribute_kv WHERE " + From 56f9dce24244b4e8c6d7b516854bc19a65529100 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 9 Jan 2024 12:27:40 +0200 Subject: [PATCH 11/17] updated BaseAttributeService, InternalTelemetryService, RuleEngineTelemetryService to store old methods for backward compatibility --- .../main/data/upgrade/3.6.3/schema_update.sql | 100 +++++------------- .../controller/TelemetryController.java | 2 +- .../device/ClaimDevicesServiceImpl.java | 5 +- .../service/edge/rpc/EdgeGrpcService.java | 5 +- .../telemetry/BaseTelemetryProcessor.java | 2 +- .../DefaultTbEntityViewService.java | 5 +- .../install/SqlDatabaseUpgradeService.java | 60 +---------- .../ota/DefaultOtaPackageStateService.java | 5 +- .../state/DefaultDeviceStateService.java | 4 +- .../DefaultTbLocalSubscriptionService.java | 4 +- .../TbAttributeSubscriptionScope.java | 31 +++++- .../csv/AbstractBulkImportService.java | 3 +- .../DefaultTelemetrySubscriptionService.java | 95 ++++++++++++++++- .../telemetry/InternalTelemetryService.java | 7 ++ .../state/DefaultDeviceStateServiceTest.java | 23 ++-- .../dao/attributes/AttributesService.java | 22 ++++ .../server/dao/attributes/AttributeUtils.java | 6 ++ .../dao/attributes/BaseAttributesService.java | 57 +++++++++- .../attributes/CachedAttributesService.java | 36 +++++++ .../main/resources/sql/schema-entities.sql | 2 +- .../main/resources/sql/schema-timescale.sql | 2 +- dao/src/main/resources/sql/schema-ts-psql.sql | 2 +- .../api/RuleEngineTelemetryService.java | 37 +++++++ .../TbCopyAttributesToEntityViewNode.java | 13 +-- .../rule/engine/math/TbMathNode.java | 5 +- .../engine/telemetry/TbMsgAttributesNode.java | 18 ++-- .../telemetry/TbMsgDeleteAttributesNode.java | 16 +-- .../rule/engine/math/TbMathNodeTest.java | 4 +- .../rule/engine/profile/DeviceStateTest.java | 3 +- .../telemetry/TbMsgAttributesNodeTest.java | 9 +- .../TbMsgDeleteAttributesNodeTest.java | 3 +- 31 files changed, 380 insertions(+), 206 deletions(-) diff --git a/application/src/main/data/upgrade/3.6.3/schema_update.sql b/application/src/main/data/upgrade/3.6.3/schema_update.sql index 30d9fa3adb..e11c52f77f 100644 --- a/application/src/main/data/upgrade/3.6.3/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.3/schema_update.sql @@ -21,14 +21,11 @@ $$ BEGIN -- in case of running the upgrade script a second time: IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'attribute_kv' and column_name='entity_type') THEN - IF EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_attribute_kv_by_key_and_last_update_ts') THEN - ALTER INDEX idx_attribute_kv_by_key_and_last_update_ts RENAME TO idx_attribute_kv_by_key_and_last_update_ts_old; - END IF; + ALTER INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts RENAME TO idx_attribute_kv_by_key_and_last_update_ts_old; IF EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'attribute_kv_pkey') THEN ALTER TABLE attribute_kv RENAME CONSTRAINT attribute_kv_pkey TO attribute_kv_pkey_old; END IF; - ALTER TABLE attribute_kv - RENAME TO attribute_kv_old; + ALTER TABLE attribute_kv RENAME TO attribute_kv_old; CREATE TABLE IF NOT EXISTS attribute_kv ( entity_id uuid, @@ -43,6 +40,8 @@ $$ CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); END IF; + DROP VIEW IF EXISTS device_info_view; + DROP VIEW IF EXISTS device_info_active_attribute_view; END; $$; @@ -51,14 +50,12 @@ DO $$ BEGIN IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'ts_kv_dictionary') THEN - ALTER TABLE ts_kv_dictionary - RENAME CONSTRAINT ts_key_id_pkey TO key_id_pkey; - ALTER TABLE ts_kv_dictionary - RENAME TO key_dictionary; + ALTER TABLE ts_kv_dictionary RENAME CONSTRAINT ts_key_id_pkey TO key_dictionary_id_pkey; + ALTER TABLE ts_kv_dictionary RENAME TO key_dictionary; ELSE CREATE TABLE IF NOT EXISTS key_dictionary( key varchar(255) NOT NULL, key_id serial UNIQUE, - CONSTRAINT key_id_pkey PRIMARY KEY (key) + CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); END IF; END; @@ -83,83 +80,40 @@ $$ LANGUAGE plpgsql; -- insert keys into key_dictionary DO $$ -DECLARE - insert_record RECORD; - key_cursor refcursor; -BEGIN - IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN - OPEN key_cursor FOR SELECT DISTINCT attribute_key - FROM attribute_kv_old - ORDER BY attribute_key; - LOOP - FETCH key_cursor INTO insert_record; - EXIT WHEN NOT FOUND; - IF NOT EXISTS(SELECT key FROM key_dictionary WHERE key = insert_record.attribute_key) THEN - INSERT INTO key_dictionary(key) VALUES (insert_record.attribute_key); - END IF; - END LOOP; - CLOSE key_cursor; - END IF; -END; + BEGIN + IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN + INSERT INTO key_dictionary(key) SELECT DISTINCT attribute_key FROM attribute_kv_old ON CONFLICT DO NOTHING; + END IF; + END; $$; --- create procedure to migrate all rows from attribute_kv_old to attribute_kv -CREATE OR REPLACE PROCEDURE insert_into_attribute_kv(IN path_to_file varchar) - LANGUAGE plpgsql AS +-- migrate attributes from attribute_kv_old to attribute_kv +DO $$ DECLARE row_num_old integer; row_num integer; - attribute_scope_array text[]; BEGIN - attribute_scope_array := ARRAY['SERVER_SCOPE', 'CLIENT_SCOPE', 'SHARED_SCOPE']; IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN - EXECUTE format('COPY (SELECT records.entity_id AS entity_id, - to_attribute_type_id(records.attribute_type) AS attribute_type, - records.attribute_key AS attribute_key, - records.bool_v AS bool_v, - records.str_v AS str_v, - records.long_v AS long_v, - records.dbl_v AS dbl_v, - records.json_v AS json_v, - records.last_update_ts AS last_update_ts - FROM (SELECT entity_id, - attribute_type, - key_id AS attribute_key, - bool_v, - str_v, - long_v, - dbl_v, - json_v, - last_update_ts - FROM attribute_kv_old INNER JOIN key_dictionary ON (attribute_kv_old.attribute_key = key_dictionary.key) - WHERE attribute_type= ANY(%L)) AS records) TO %L;', attribute_scope_array, path_to_file); - EXECUTE format('COPY attribute_kv FROM %L', path_to_file); + INSERT INTO attribute_kv(entity_id, attribute_type, attribute_key, bool_v, str_v, long_v, dbl_v, json_v, last_update_ts) + SELECT a.entity_id, to_attribute_type_id(a.attribute_type), k.key_id, a.bool_v, a.str_v, a.long_v, a.dbl_v, a.json_v, a.last_update_ts + FROM attribute_kv_old a INNER JOIN key_dictionary k ON (a.attribute_key = k.key) + WHERE a.attribute_type IN ('SERVER_SCOPE', 'CLIENT_SCOPE', 'SHARED_SCOPE'); SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; SELECT COUNT(*) INTO row_num FROM attribute_kv; RAISE NOTICE 'Migrated % of % rows', row_num, row_num_old; + + IF row_num != 0 THEN + DROP TABLE IF EXISTS attribute_kv_old; + ELSE + RAISE EXCEPTION 'Table attribute_kv is empty'; + END IF; + + CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); END IF; EXCEPTION WHEN others THEN ROLLBACK; RAISE EXCEPTION 'Error during COPY: %', SQLERRM; END -$$; - -CREATE OR REPLACE PROCEDURE drop_attribute_kv_old_table() - LANGUAGE plpgsql AS -$$ -DECLARE - row_num integer; -BEGIN - SELECT COUNT(*) INTO row_num FROM attribute_kv; - IF row_num != 0 then - DROP TABLE IF EXISTS attribute_kv_old; - DROP PROCEDURE IF EXISTS insert_into_attribute_kv(IN path_to_file varchar); - ELSE - RAISE EXCEPTION 'Table attribute_kv is empty'; - END IF; - RETURN; -END; -$$; - +$$; \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index e041975a15..3bde00c5f9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -631,7 +631,7 @@ public class TelemetryController extends BaseController { } SecurityUser user = getCurrentUser(); return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.WRITE_ATTRIBUTES, entityIdSrc, (result, tenantId, entityId) -> { - tsSubService.saveAndNotify(tenantId, entityId, scope.name(), attributes, new FutureCallback() { + tsSubService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { logAttributesUpdated(user, entityId, scope, attributes, null); diff --git a/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java index 08852be7fa..882f5fade2 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/ClaimDevicesServiceImpl.java @@ -31,7 +31,6 @@ import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.CustomerId; @@ -187,7 +186,7 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService { } SettableFuture result = SettableFuture.create(); telemetryService.saveAndNotify( - tenantId, savedDevice.getId(), DataConstants.SERVER_SCOPE, Collections.singletonList( + tenantId, savedDevice.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList( new BaseAttributeKvEntry(new BooleanDataEntry(CLAIM_ATTRIBUTE_NAME, true), System.currentTimeMillis()) ), new FutureCallback<>() { @@ -230,7 +229,7 @@ public class ClaimDevicesServiceImpl implements ClaimDevicesService { } SettableFuture result = SettableFuture.create(); telemetryService.deleteAndNotify(device.getTenantId(), - device.getId(), DataConstants.SERVER_SCOPE, Arrays.asList(CLAIM_ATTRIBUTE_NAME, CLAIM_DATA_ATTRIBUTE_NAME), new FutureCallback<>() { + device.getId(), AttributeScope.SERVER_SCOPE, Arrays.asList(CLAIM_ATTRIBUTE_NAME, CLAIM_DATA_ATTRIBUTE_NAME), new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) { result.set(tmp); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 89cae58264..b57c85d982 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -29,6 +29,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.ResourceUtils; import org.thingsboard.server.common.data.edge.Edge; @@ -412,7 +413,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new LongDataEntry(key, value))), new AttributeSaveCallback(tenantId, edgeId, key, value)); } else { - tsSubService.saveAttrAndNotify(tenantId, edgeId, DataConstants.SERVER_SCOPE, key, value, new AttributeSaveCallback(tenantId, edgeId, key, value)); + tsSubService.saveAttrAndNotify(tenantId, edgeId, AttributeScope.SERVER_SCOPE, key, value, new AttributeSaveCallback(tenantId, edgeId, key, value)); } } @@ -424,7 +425,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), new AttributeSaveCallback(tenantId, edgeId, key, value)); } else { - tsSubService.saveAttrAndNotify(tenantId, edgeId, DataConstants.SERVER_SCOPE, key, value, new AttributeSaveCallback(tenantId, edgeId, key, value)); + tsSubService.saveAttrAndNotify(tenantId, edgeId, AttributeScope.SERVER_SCOPE, key, value, new AttributeSaveCallback(tenantId, edgeId, key, value)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 51f3fac9f2..327048f27a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -252,7 +252,7 @@ public abstract class BaseTelemetryProcessor extends BaseEdgeProcessor { JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); List attributes = new ArrayList<>(JsonConverter.convertToAttributes(json)); String scope = metaData.getValue("scope"); - tsSubService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback() { + tsSubService.saveAndNotify(tenantId, entityId, AttributeScope.valueOf(scope), attributes, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { var defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index 665735cc22..a214a88d3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -26,7 +26,6 @@ import org.springframework.stereotype.Service; import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.User; @@ -288,7 +287,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen (startTime == 0 && endTime > lastUpdateTs) || (startTime < lastUpdateTs && endTime > lastUpdateTs); }).collect(Collectors.toList()); - tsSubService.saveAndNotify(entityView.getTenantId(), entityId, scope.name(), attributes, new FutureCallback() { + tsSubService.saveAndNotify(entityView.getTenantId(), entityId, scope, attributes, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { try { @@ -356,7 +355,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen EntityViewId entityId = entityView.getId(); SettableFuture resultFuture = SettableFuture.create(); if (keys != null && !keys.isEmpty()) { - tsSubService.deleteAndNotify(entityView.getTenantId(), entityId, scope.name(), keys, new FutureCallback() { + tsSubService.deleteAndNotify(entityView.getTenantId(), entityId, scope, keys, new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index e3927b376a..cbd8686918 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -779,31 +779,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService }); break; case "3.6.3": - updateSchema("3.6.3", 3006003, "3.7.0", 3007000, connection -> { - try { - Path pathToTempAttributeKvFile; - if (SystemUtils.IS_OS_WINDOWS) { - pathToTempAttributeKvFile = createTempFileWindows("attribute_kv_temp",".sql"); - } else { - pathToTempAttributeKvFile = createTempFile("attribute_kv", "attribute_kv_temp.sql"); - } - executeQuery(connection, "call insert_into_attribute_kv('" + pathToTempAttributeKvFile + "')"); - - // remove attribute_kv_old - executeQuery(connection, "call drop_attribute_kv_old_table()"); - - //create index for new table attribute_kv - executeQuery(connection, "CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc);"); - - // remove temp files - boolean deleteTsKvFile = Files.deleteIfExists(pathToTempAttributeKvFile); - if (deleteTsKvFile) { - log.info("Successfully deleted the temp file for attribute_kv table upgrade!"); - } - } catch (Exception e) { - log.error("Failed updating schema!!!", e); - } - }); + updateSchema("3.6.3", 3006003, "3.7.0", 3007000, null); break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); @@ -829,40 +805,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } - private static Path createTempFile(String tempDirectoryName, String tempFileName) throws IOException { - Path pathToTempAttributeKvFile; - Path tempDirPath = Files.createTempDirectory(tempDirectoryName); - File tempDirAsFile = tempDirPath.toFile(); - boolean writable = tempDirAsFile.setWritable(true, false); - boolean readable = tempDirAsFile.setReadable(true, false); - boolean executable = tempDirAsFile.setExecutable(true, false); - pathToTempAttributeKvFile = tempDirPath.resolve(tempFileName).toAbsolutePath(); - - if (!(writable && readable && executable)) { - throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!"); - } - return pathToTempAttributeKvFile; - } - - private static Path createTempFileWindows(String prefix, String suffix) throws IOException { - Path pathToTempAttributeKvFile; - log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR); - Path pathToDir; - String thingsboardWindowsUpgradeDir = System.getenv("THINGSBOARD_WINDOWS_UPGRADE_DIR"); - if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) { - log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR); - pathToDir = Paths.get(thingsboardWindowsUpgradeDir); - } else { - log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR); - pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER); - } - log.info("Directory: {} will be used for creation temporary upgrade files!", pathToDir); - - Path attributeKvFile = Files.createTempFile(pathToDir, prefix, suffix); - pathToTempAttributeKvFile = attributeKvFile.toAbsolutePath(); - return pathToTempAttributeKvFile; - } - private void runSchemaUpdateScript(Connection connection, String version) throws Exception { Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, SCHEMA_UPDATE_SQL); loadSql(schemaUpdateFile, connection); diff --git a/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java b/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java index 70c6655620..adaa6012de 100644 --- a/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/ota/DefaultOtaPackageStateService.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; @@ -335,7 +336,7 @@ public class DefaultOtaPackageStateService implements OtaPackageStateService { remove(device, otaPackageType, attrToRemove); - telemetryService.saveAndNotify(tenantId, deviceId, DataConstants.SHARED_SCOPE, attributes, new FutureCallback<>() { + telemetryService.saveAndNotify(tenantId, deviceId, AttributeScope.SHARED_SCOPE, attributes, new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) { log.trace("[{}] Success save attributes with target firmware!", deviceId); @@ -353,7 +354,7 @@ public class DefaultOtaPackageStateService implements OtaPackageStateService { } private void remove(Device device, OtaPackageType otaPackageType, List attributesKeys) { - telemetryService.deleteAndNotify(device.getTenantId(), device.getId(), DataConstants.SHARED_SCOPE, attributesKeys, + telemetryService.deleteAndNotify(device.getTenantId(), device.getId(), AttributeScope.SHARED_SCOPE, attributesKeys, new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) { diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 0affb51fc4..d127259099 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -806,7 +806,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService(deviceId, key, value)); } else { - tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value)); + tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value)); } } @@ -817,7 +817,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService(deviceId, key, value)); } else { - tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value)); + tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index a6d5716d2b..c2b218197d 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -449,8 +449,8 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } final Map keyStates = subscription.getKeyStates(); AttributeScope scope; - if (subscription.getScope() != null && !TbAttributeSubscriptionScope.ANY_SCOPE.equals(subscription.getScope())) { - scope = AttributeScope.valueOf(subscription.getScope().name()); + if (subscription.getScope() != null && subscription.getScope().getAttributeScope() != null) { + scope = subscription.getScope().getAttributeScope(); } else { scope = AttributeScope.CLIENT_SCOPE; } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java index b3874e28fb..ed2558dd42 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAttributeSubscriptionScope.java @@ -15,8 +15,37 @@ */ package org.thingsboard.server.service.subscription; +import org.thingsboard.server.common.data.AttributeScope; + public enum TbAttributeSubscriptionScope { - ANY_SCOPE, CLIENT_SCOPE, SHARED_SCOPE, SERVER_SCOPE + ANY_SCOPE(), + CLIENT_SCOPE(AttributeScope.CLIENT_SCOPE), + SHARED_SCOPE(AttributeScope.SHARED_SCOPE), + SERVER_SCOPE(AttributeScope.SERVER_SCOPE); + + private final AttributeScope attributeScope; + + TbAttributeSubscriptionScope() { + this.attributeScope = null; + } + + TbAttributeSubscriptionScope(AttributeScope attributeScope) { + this.attributeScope = attributeScope; + } + + public AttributeScope getAttributeScope() { + return attributeScope; + } + + public static TbAttributeSubscriptionScope of(AttributeScope attributeScope) { + for (TbAttributeSubscriptionScope scope : TbAttributeSubscriptionScope.values()) { + if (attributeScope == scope.getAttributeScope()) { + return scope; + } + } + throw new IllegalArgumentException("Unknown AttributeScope: " + attributeScope.name()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 68c0347b40..5b2e458c59 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -29,6 +29,7 @@ import org.springframework.security.core.context.SecurityContextHolder; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasAdditionalInfo; import org.thingsboard.server.common.data.HasTenantId; @@ -230,7 +231,7 @@ public abstract class AbstractBulkImportService attributes = new ArrayList<>(JsonConverter.convertToAttributes(kvsEntry.getValue())); accessValidator.validateEntityAndCallback(user, Operation.WRITE_ATTRIBUTES, entity.getId(), (result, tenantId, entityId) -> { - tsSubscriptionService.saveAndNotify(tenantId, entityId, scope, attributes, new FutureCallback<>() { + tsSubscriptionService.saveAndNotify(tenantId, entityId, AttributeScope.valueOf(scope), attributes, new FutureCallback<>() { @Override public void onSuccess(Void unused) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index b01b103007..531f518579 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -242,19 +242,38 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer saveAndNotify(tenantId, entityId, scope, attributes, true, callback); } + @Override + + public void saveAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, FutureCallback callback) { + saveAndNotify(tenantId, entityId, scope, attributes, true, callback); + } + @Override public void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { checkInternalEntity(entityId); saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback); } + @Override + public void saveAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, boolean notifyDevice, FutureCallback callback) { + checkInternalEntity(entityId); + saveAndNotifyInternal(tenantId, entityId, scope, attributes, notifyDevice, callback); + } + @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { - ListenableFuture> saveFuture = attrService.save(tenantId, entityId, AttributeScope.valueOf(scope), attributes); + ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } + @Override + public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, boolean notifyDevice, FutureCallback callback) { + ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); + addVoidCallback(saveFuture, callback); + addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); + } + @Override public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { checkInternalEntity(entityId); @@ -274,19 +293,38 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer deleteAndNotifyInternal(tenantId, entityId, scope, keys, false, callback); } + @Override + public void deleteAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List keys, FutureCallback callback) { + checkInternalEntity(entityId); + deleteAndNotifyInternal(tenantId, entityId, scope, keys, false, callback); + } + @Override public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, FutureCallback callback) { checkInternalEntity(entityId); deleteAndNotifyInternal(tenantId, entityId, scope, keys, notifyDevice, callback); } + @Override + public void deleteAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List keys, boolean notifyDevice, FutureCallback callback) { + checkInternalEntity(entityId); + deleteAndNotifyInternal(tenantId, entityId, scope, keys, notifyDevice, callback); + } + @Override public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, FutureCallback callback) { - ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), keys); + ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); addVoidCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice)); } + @Override + public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, AttributeScope scope, List keys, boolean notifyDevice, FutureCallback callback) { + ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); + addVoidCallback(deleteFuture, callback); + addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope.name(), keys, notifyDevice)); + } + @Override public void deleteLatest(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback) { checkInternalEntity(entityId); @@ -328,24 +366,49 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer , System.currentTimeMillis())), callback); } + + @Override + public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, long value, FutureCallback callback) { + saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new LongDataEntry(key, value) + , System.currentTimeMillis())), callback); + } + @Override public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, String value, FutureCallback callback) { saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(key, value) , System.currentTimeMillis())), callback); } + @Override + public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, String value, FutureCallback callback) { + saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(key, value) + , System.currentTimeMillis())), callback); + } + @Override public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, double value, FutureCallback callback) { saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new DoubleDataEntry(key, value) , System.currentTimeMillis())), callback); } + @Override + public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, double value, FutureCallback callback) { + saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new DoubleDataEntry(key, value) + , System.currentTimeMillis())), callback); + } + @Override public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback) { saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) , System.currentTimeMillis())), callback); } + @Override + public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, boolean value, FutureCallback callback) { + saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) + , System.currentTimeMillis())), callback); + } + @Override public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, long value) { SettableFuture future = SettableFuture.create(); @@ -353,6 +416,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer return future; } + @Override + public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, long value) { + SettableFuture future = SettableFuture.create(); + saveAttrAndNotify(tenantId, entityId, scope, key, value, new VoidFutureCallback(future)); + return future; + } + @Override public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, String value) { SettableFuture future = SettableFuture.create(); @@ -360,6 +430,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer return future; } + @Override + public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, String value) { + SettableFuture future = SettableFuture.create(); + saveAttrAndNotify(tenantId, entityId, scope, key, value, new VoidFutureCallback(future)); + return future; + } + @Override public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, double value) { SettableFuture future = SettableFuture.create(); @@ -367,6 +444,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer return future; } + @Override + public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, double value) { + SettableFuture future = SettableFuture.create(); + saveAttrAndNotify(tenantId, entityId, scope, key, value, new VoidFutureCallback(future)); + return future; + } + @Override public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value) { SettableFuture future = SettableFuture.create(); @@ -374,6 +458,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer return future; } + @Override + public ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, boolean value) { + SettableFuture future = SettableFuture.create(); + saveAttrAndNotify(tenantId, entityId, scope, key, value, new VoidFutureCallback(future)); + return future; + } + private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice) { forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 301376b16a..5d451d4b32 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -33,12 +34,18 @@ public interface InternalTelemetryService extends RuleEngineTelemetryService { void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback); + @Deprecated(since = "3.7.0") void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); + void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, boolean notifyDevice, FutureCallback callback); + void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + @Deprecated(since = "3.7.0") void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, FutureCallback callback); + void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, AttributeScope scope, List keys, boolean notifyDevice, FutureCallback callback); + void deleteLatestInternal(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback); } diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 05624b8741..8bba931a0a 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -26,6 +26,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -251,7 +252,7 @@ public class DefaultDeviceStateServiceTest { long newTimeout = System.currentTimeMillis() - deviceState.getLastActivityTime() + increase; service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(), eq(ACTIVITY_STATE), any(), any()); + verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(AttributeScope.class), eq(ACTIVITY_STATE), any(), any()); Thread.sleep(defaultTimeout + increase); service.checkStates(); activityVerify(false); @@ -290,7 +291,7 @@ public class DefaultDeviceStateServiceTest { long newTimeout = 1; Thread.sleep(newTimeout); - verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(), eq(ACTIVITY_STATE), any(), any()); + verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(AttributeScope.class), eq(ACTIVITY_STATE), any(), any()); } @Test @@ -311,7 +312,7 @@ public class DefaultDeviceStateServiceTest { service.onDeviceActivity(tenantId, deviceId, System.currentTimeMillis()); activityVerify(true); - verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(), eq(ACTIVITY_STATE), any(), any()); + verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(AttributeScope.class), eq(ACTIVITY_STATE), any(), any()); long newTimeout = 1; Thread.sleep(newTimeout); @@ -352,11 +353,11 @@ public class DefaultDeviceStateServiceTest { long newTimeout = 1; service.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, newTimeout); - verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(), eq(ACTIVITY_STATE), any(), any()); + verify(telemetrySubscriptionService, never()).saveAttrAndNotify(any(), eq(deviceId), any(AttributeScope.class), eq(ACTIVITY_STATE), any(), any()); } private void activityVerify(boolean isActive) { - verify(telemetrySubscriptionService, times(1)).saveAttrAndNotify(any(), eq(deviceId), any(), eq(ACTIVITY_STATE), eq(isActive), any()); + verify(telemetrySubscriptionService, times(1)).saveAttrAndNotify(any(), eq(deviceId), any(AttributeScope.class), eq(ACTIVITY_STATE), eq(isActive), any()); } @Test @@ -403,19 +404,19 @@ public class DefaultDeviceStateServiceTest { assertThat(deviceState.isActive()).isEqualTo(true); assertThat(deviceState.getLastActivityTime()).isEqualTo(lastReportedActivity); then(telemetrySubscriptionService).should().saveAttrAndNotify( - any(), eq(deviceId), any(), eq(LAST_ACTIVITY_TIME), eq(lastReportedActivity), any() + any(), eq(deviceId), any(AttributeScope.class), eq(LAST_ACTIVITY_TIME), eq(lastReportedActivity), any() ); assertThat(deviceState.getLastInactivityAlarmTime()).isEqualTo(expectedInactivityAlarmTime); if (shouldSetInactivityAlarmTimeToZero) { then(telemetrySubscriptionService).should().saveAttrAndNotify( - any(), eq(deviceId), any(), eq(INACTIVITY_ALARM_TIME), eq(0L), any() + any(), eq(deviceId), any(AttributeScope.class), eq(INACTIVITY_ALARM_TIME), eq(0L), any() ); } if (shouldUpdateActivityStateToActive) { then(telemetrySubscriptionService).should().saveAttrAndNotify( - eq(TenantId.SYS_TENANT_ID), eq(deviceId), eq(SERVER_SCOPE), eq(ACTIVITY_STATE), eq(true), any() + eq(TenantId.SYS_TENANT_ID), eq(deviceId), eq(AttributeScope.SERVER_SCOPE), eq(ACTIVITY_STATE), eq(true), any() ); var msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -497,7 +498,7 @@ public class DefaultDeviceStateServiceTest { assertThat(deviceState.isActive()).isEqualTo(expectedActivityState); if (activityState && !expectedActivityState) { then(telemetrySubscriptionService).should().saveAttrAndNotify( - any(), eq(deviceId), any(), eq(ACTIVITY_STATE), eq(false), any() + any(), eq(deviceId), any(AttributeScope.class), eq(ACTIVITY_STATE), eq(false), any() ); } } @@ -594,7 +595,7 @@ public class DefaultDeviceStateServiceTest { if (shouldUpdateActivityStateToInactive) { then(telemetrySubscriptionService).should().saveAttrAndNotify( - eq(TenantId.SYS_TENANT_ID), eq(deviceId), eq(SERVER_SCOPE), eq(ACTIVITY_STATE), eq(false), any() + eq(TenantId.SYS_TENANT_ID), eq(deviceId), eq(AttributeScope.SERVER_SCOPE), eq(ACTIVITY_STATE), eq(false), any() ); var msgCaptor = ArgumentCaptor.forClass(TbMsg.class); @@ -611,7 +612,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.isActive()).isFalse(); then(telemetrySubscriptionService).should().saveAttrAndNotify( - eq(TenantId.SYS_TENANT_ID), eq(deviceId), eq(SERVER_SCOPE), + eq(TenantId.SYS_TENANT_ID), eq(deviceId), eq(AttributeScope.SERVER_SCOPE), eq(INACTIVITY_ALARM_TIME), eq(expectedLastInactivityAlarmTime), any() ); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index ccbc90825a..d7c6f5981e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -31,20 +32,41 @@ import java.util.Optional; */ public interface AttributesService { + @Deprecated(since = "3.7.0") + ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey); + ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey); + @Deprecated(since = "3.7.0") + ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys); + ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, Collection attributeKeys); + @Deprecated(since = "3.7.0") + ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope); + ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope); + @Deprecated(since = "3.7.0") + ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes); + ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); + @Deprecated(since = "3.7.0") + ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); + @Deprecated(since = "3.7.0") + ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys); + ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys); List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); + @Deprecated(since = "3.7.0") + List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds); + List findAllKeysByEntityIds(TenantId tenantId, List entityIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java index 983661ff6e..7077edac40 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java @@ -26,6 +26,12 @@ import java.util.List; public class AttributeUtils { + @Deprecated(since = "3.7.0") + public static void validate(EntityId id, String scope) { + Validator.validateId(id.getId(), "Incorrect id " + id); + Validator.validateString(scope, "Incorrect scope " + scope); + } + public static void validate(EntityId id, AttributeScope scope) { Validator.validateId(id.getId(), "Incorrect id " + id); Validator.checkNotNull(scope, "Incorrect scope " + scope); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 3f8c88d609..84b504ca6e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -23,6 +23,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -53,6 +54,13 @@ public class BaseAttributesService implements AttributesService { this.attributesDao = attributesDao; } + @Override + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { + validate(entityId, scope); + Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey)); + } + @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -60,17 +68,30 @@ public class BaseAttributesService implements AttributesService { return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKey)); } + @Override + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { + validate(entityId, scope); + attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys)); + } + @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, Collection attributeKeys) { validate(entityId, scope); attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); - return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKeys)); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKeys)); + } + + @Override + public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { + validate(entityId, scope); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, AttributeScope.valueOf(scope))); } @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope) { validate(entityId, scope); - return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); } @Override @@ -78,29 +99,55 @@ public class BaseAttributesService implements AttributesService { return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId); } + @Override + public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { + return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); + } + @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { + validate(entityId, scope); + AttributeUtils.validate(attribute, valueNoXssValidation); + return attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); + } + @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, scope, attribute); + return attributesDao.save(tenantId, entityId, scope, attribute); + } + + @Override + public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + validate(entityId, scope); + AttributeUtils.validate(attributes, valueNoXssValidation); + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute)).collect(Collectors.toList()); + return Futures.allAsList(saveFutures); } @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } + @Override + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + validate(entityId, scope); + return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys)); + } + @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); + return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 6029be94c9..f87827b535 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -108,6 +109,11 @@ public class CachedAttributesService implements AttributesService { } + @Override + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { + return find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey); + } + @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -137,6 +143,11 @@ public class CachedAttributesService implements AttributesService { } } + @Override + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { + return find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); + } + @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, final Collection attributeKeysNonUnique) { validate(entityId, scope); @@ -204,6 +215,11 @@ public class CachedAttributesService implements AttributesService { return cachedAttributes; } + @Override + public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { + return findAll(tenantId, entityId, AttributeScope.valueOf(scope)); + } + @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope) { validate(entityId, scope); @@ -215,11 +231,21 @@ public class CachedAttributesService implements AttributesService { return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId); } + @Override + public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { + return findAllKeysByEntityIds(tenantId, entityIds); + } + @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { + return save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); + } + @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); @@ -228,6 +254,11 @@ public class CachedAttributesService implements AttributesService { return Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor); } + @Override + public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + return save(tenantId, entityId, scope, attributes); + } + @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); @@ -249,6 +280,11 @@ public class CachedAttributesService implements AttributesService { return key; } + @Override + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + return removeAll(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); + } + @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index c0a3815dc2..056201a56e 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -551,7 +551,7 @@ CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, - CONSTRAINT key_id_pkey PRIMARY KEY (key) + CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); CREATE TABLE IF NOT EXISTS oauth2_params ( diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 380f813680..caeadbe12e 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -31,7 +31,7 @@ CREATE TABLE IF NOT EXISTS ts_kv ( CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, - CONSTRAINT key_id_pkey PRIMARY KEY (key) + CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); CREATE TABLE IF NOT EXISTS ts_kv_latest ( diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index bd9713c104..69932e130f 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -31,7 +31,7 @@ CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, - CONSTRAINT key_id_pkey PRIMARY KEY (key) + CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); CREATE OR REPLACE PROCEDURE drop_partitions_by_system_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java index a61e83f48f..c54c6aed04 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineTelemetryService.java @@ -17,6 +17,7 @@ package org.thingsboard.rule.engine.api; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,32 +41,68 @@ public interface RuleEngineTelemetryService { void saveWithoutLatestAndNotify(TenantId tenantId, CustomerId id, EntityId entityId, List ts, long ttl, FutureCallback callback); + @Deprecated(since = "3.7.0") void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, FutureCallback callback); + void saveAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, FutureCallback callback); + + @Deprecated(since = "3.7.0") void saveAndNotify(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback); + void saveAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, boolean notifyDevice, FutureCallback callback); + void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback); + @Deprecated(since = "3.7.0") ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, long value); + ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, long value); + + @Deprecated(since = "3.7.0") ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, String value); + ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, String value); + + @Deprecated(since = "3.7.0") ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, double value); + ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, double value); + + @Deprecated(since = "3.7.0") ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value); + ListenableFuture saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, boolean value); + + @Deprecated(since = "3.7.0") void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, long value, FutureCallback callback); + void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, long value, FutureCallback callback); + + @Deprecated(since = "3.7.0") void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, String value, FutureCallback callback); + void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, String value, FutureCallback callback); + + @Deprecated(since = "3.7.0") void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, double value, FutureCallback callback); + void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, double value, FutureCallback callback); + + @Deprecated(since = "3.7.0") void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, boolean value, FutureCallback callback); + void saveAttrAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, String key, boolean value, FutureCallback callback); + + @Deprecated(since = "3.7.0") void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback); + void deleteAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List keys, FutureCallback callback); + + @Deprecated(since = "3.7.0") void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, FutureCallback callback); + void deleteAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List keys, boolean notifyDevice, FutureCallback callback); + void deleteLatest(TenantId tenantId, EntityId entityId, List keys, FutureCallback callback); void deleteAllLatest(TenantId tenantId, EntityId entityId, FutureCallback> callback); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index b1248c0856..14b368e8dc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -29,6 +29,7 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -79,8 +80,8 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { ACTIVITY_EVENT, INACTIVITY_EVENT, POST_ATTRIBUTES_REQUEST)) { if (!msg.getMetaData().getData().isEmpty()) { long now = System.currentTimeMillis(); - String scope = msg.isTypeOf(POST_ATTRIBUTES_REQUEST) ? - DataConstants.CLIENT_SCOPE : msg.getMetaData().getValue(DataConstants.SCOPE); + AttributeScope scope = msg.isTypeOf(POST_ATTRIBUTES_REQUEST) ? + AttributeScope.CLIENT_SCOPE : AttributeScope.valueOf(msg.getMetaData().getValue(DataConstants.SCOPE)); ListenableFuture> entityViewsFuture = ctx.getEntityViewService().findEntityViewsByTenantIdAndEntityIdAsync(ctx.getTenantId(), msg.getOriginator()); @@ -145,17 +146,17 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { ctx.enqueueForTellNext(ctx.newMsg(msg.getQueueName(), msg.getType(), entityView.getId(), msg.getCustomerId(), msg.getMetaData(), msg.getData()), SUCCESS); } - private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { + private boolean attributeContainsInEntityView(AttributeScope scope, String attrKey, EntityView entityView) { AttributesEntityView attributesEntityView = entityView.getKeys().getAttributes(); List keys = null; switch (scope) { - case DataConstants.CLIENT_SCOPE: + case CLIENT_SCOPE: keys = attributesEntityView.getCs(); break; - case DataConstants.SERVER_SCOPE: + case SERVER_SCOPE: keys = attributesEntityView.getSs(); break; - case DataConstants.SHARED_SCOPE: + case SHARED_SCOPE: keys = attributesEntityView.getSh(); break; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java index 455e5c8da7..4f25e0760f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java @@ -34,7 +34,6 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -213,11 +212,11 @@ public class TbMathNode implements TbNode { if (isIntegerResult(mathResultDef, config.getOperation())) { var value = toIntValue(result); return ctx.getTelemetryService().saveAttrAndNotify( - ctx.getTenantId(), msg.getOriginator(), attributeScope.name(), mathResultDef.getKey(), value); + ctx.getTenantId(), msg.getOriginator(), attributeScope, mathResultDef.getKey(), value); } else { var value = toDoubleValue(mathResultDef, result); return ctx.getTelemetryService().saveAttrAndNotify( - ctx.getTenantId(), msg.getOriginator(), attributeScope.name(), mathResultDef.getKey(), value); + ctx.getTenantId(), msg.getOriginator(), attributeScope, mathResultDef.getKey(), value); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 5fa7f91b9f..c29dd67915 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -90,7 +90,7 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellSuccess(msg); return; } - String scope = getScope(msg.getMetaData().getValue(SCOPE)); + AttributeScope scope = getScope(msg.getMetaData().getValue(SCOPE)); boolean sendAttributesUpdateNotification = checkSendNotification(scope); if (!config.isUpdateAttributesOnlyOnValueChange()) { @@ -99,7 +99,7 @@ public class TbMsgAttributesNode implements TbNode { } List keys = newAttributes.stream().map(KvEntry::getKey).collect(Collectors.toList()); - ListenableFuture> findFuture = ctx.getAttributesService().find(ctx.getTenantId(), msg.getOriginator(), AttributeScope.valueOf(scope), keys); + ListenableFuture> findFuture = ctx.getAttributesService().find(ctx.getTenantId(), msg.getOriginator(), scope, keys); DonAsynchron.withCallback(findFuture, currentAttributes -> { @@ -110,7 +110,7 @@ public class TbMsgAttributesNode implements TbNode { MoreExecutors.directExecutor()); } - void saveAttr(List attributes, TbContext ctx, TbMsg msg, String scope, boolean sendAttributesUpdateNotification) { + void saveAttr(List attributes, TbContext ctx, TbMsg msg, AttributeScope scope, boolean sendAttributesUpdateNotification) { if (attributes.isEmpty()) { ctx.tellSuccess(msg); return; @@ -122,7 +122,7 @@ public class TbMsgAttributesNode implements TbNode { attributes, config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY)), sendAttributesUpdateNotification ? - new AttributesUpdateNodeCallback(ctx, msg, scope, attributes) : + new AttributesUpdateNodeCallback(ctx, msg, scope.name(), attributes) : new TelemetryNodeCallback(ctx, msg) ); } @@ -145,8 +145,8 @@ public class TbMsgAttributesNode implements TbNode { .collect(Collectors.toList()); } - private boolean checkSendNotification(String scope) { - return config.isSendAttributesUpdatedNotification() && !CLIENT_SCOPE.equals(scope); + private boolean checkSendNotification(AttributeScope scope) { + return config.isSendAttributesUpdatedNotification() && AttributeScope.CLIENT_SCOPE != scope; } private boolean checkNotifyDeviceMdValue(String notifyDeviceMdValue) { @@ -154,11 +154,11 @@ public class TbMsgAttributesNode implements TbNode { return StringUtils.isEmpty(notifyDeviceMdValue) || Boolean.parseBoolean(notifyDeviceMdValue); } - private String getScope(String mdScopeValue) { + private AttributeScope getScope(String mdScopeValue) { if (StringUtils.isNotEmpty(mdScopeValue)) { - return mdScopeValue; + return AttributeScope.valueOf(mdScopeValue); } - return config.getScope(); + return AttributeScope.valueOf(config.getScope()); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java index 112a3c26dd..7d61d7782e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNode.java @@ -22,6 +22,7 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; @@ -32,7 +33,6 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.NOTIFY_DEVICE_METADATA_KEY; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; @Slf4j @RuleNode( @@ -69,7 +69,7 @@ public class TbMsgDeleteAttributesNode implements TbNode { if (keysToDelete.isEmpty()) { ctx.tellSuccess(msg); } else { - String scope = getScope(msg.getMetaData().getValue(SCOPE)); + AttributeScope scope = getScope(msg.getMetaData().getValue(SCOPE)); ctx.getTelemetryService().deleteAndNotify( ctx.getTenantId(), msg.getOriginator(), @@ -77,21 +77,21 @@ public class TbMsgDeleteAttributesNode implements TbNode { keysToDelete, checkNotifyDevice(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY), scope), config.isSendAttributesDeletedNotification() ? - new AttributesDeleteNodeCallback(ctx, msg, scope, keysToDelete) : + new AttributesDeleteNodeCallback(ctx, msg, scope.name(), keysToDelete) : new TelemetryNodeCallback(ctx, msg) ); } } - private String getScope(String mdScopeValue) { + private AttributeScope getScope(String mdScopeValue) { if (StringUtils.isNotEmpty(mdScopeValue)) { - return mdScopeValue; + return AttributeScope.valueOf(mdScopeValue); } - return config.getScope(); + return AttributeScope.valueOf(config.getScope()); } - private boolean checkNotifyDevice(String notifyDeviceMdValue, String scope) { - return SHARED_SCOPE.equals(scope) && (config.isNotifyDevice() || Boolean.parseBoolean(notifyDeviceMdValue)); + private boolean checkNotifyDevice(String notifyDeviceMdValue, AttributeScope scope) { + return (AttributeScope.SHARED_SCOPE == scope) && (config.isNotifyDevice() || Boolean.parseBoolean(notifyDeviceMdValue)); } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 8ee70f23d7..c4e7c46f3e 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -438,14 +438,14 @@ public class TbMathNodeTest { TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, originator, TbMsgMetaData.EMPTY, JacksonUtil.newObjectNode().put("a", 5).toString()); - when(telemetryService.saveAttrAndNotify(any(), any(), anyString(), anyString(), anyDouble())) + when(telemetryService.saveAttrAndNotify(any(), any(), any(AttributeScope.class), anyString(), anyDouble())) .thenReturn(Futures.immediateFuture(null)); node.onMsg(ctx, msg); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); - verify(telemetryService, times(1)).saveAttrAndNotify(any(), any(), anyString(), anyString(), anyDouble()); + verify(telemetryService, times(1)).saveAttrAndNotify(any(), any(), any(AttributeScope.class), anyString(), anyDouble()); TbMsg resultMsg = msgCaptor.getValue(); assertNotNull(resultMsg); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java index eef77a5304..e523313a63 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java @@ -22,6 +22,7 @@ import org.mockito.ArgumentCaptor; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @@ -75,7 +76,7 @@ public class DeviceStateTest { when(ctx.getDeviceService()).thenReturn(mock(DeviceService.class)); AttributesService attributesService = mock(AttributesService.class); - when(attributesService.find(any(), any(), any(), anyCollection())).thenReturn(Futures.immediateFuture(Collections.emptyList())); + when(attributesService.find(any(), any(), any(AttributeScope.class), anyCollection())).thenReturn(Futures.immediateFuture(Collections.emptyList())); when(ctx.getAttributesService()).thenReturn(attributesService); RuleEngineAlarmService alarmService = mock(RuleEngineAlarmService.class); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java index fe2edc941b..3efbb2f009 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java @@ -29,7 +29,7 @@ import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -54,7 +54,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.willCallRealMethod; import static org.mockito.Mockito.mock; @@ -157,7 +156,7 @@ class TbMsgAttributesNodeTest { when(ctxMock.getTenantId()).thenReturn(tenantId); when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); willCallRealMethod().given(node).init(any(TbContext.class), any(TbNodeConfiguration.class)); - willCallRealMethod().given(node).saveAttr(any(), eq(ctxMock), any(TbMsg.class), anyString(), anyBoolean()); + willCallRealMethod().given(node).saveAttr(any(), eq(ctxMock), any(TbMsg.class), any(AttributeScope.class), anyBoolean()); node.init(ctxMock, tbNodeConfiguration); @@ -169,12 +168,12 @@ class TbMsgAttributesNodeTest { var testTbMsg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, deviceId, md, TbMsg.EMPTY_STRING); List testAttrList = List.of(new BaseAttributeKvEntry(0L, new StringDataEntry("testKey", "testValue"))); - node.saveAttr(testAttrList, ctxMock, testTbMsg, DataConstants.SHARED_SCOPE, false); + node.saveAttr(testAttrList, ctxMock, testTbMsg, AttributeScope.SHARED_SCOPE, false); ArgumentCaptor notifyDeviceCaptor = ArgumentCaptor.forClass(Boolean.class); verify(telemetryServiceMock, times(1)).saveAndNotify( - eq(tenantId), eq(deviceId), eq(DataConstants.SHARED_SCOPE), + eq(tenantId), eq(deviceId), eq(AttributeScope.SHARED_SCOPE), eq(testAttrList), notifyDeviceCaptor.capture(), any() ); boolean notifyDevice = notifyDeviceCaptor.getValue(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java index 632243676f..f3fc7e6163 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java @@ -25,6 +25,7 @@ import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -81,7 +82,7 @@ public class TbMsgDeleteAttributesNodeTest { callBack.onSuccess(null); return null; }).given(telemetryService).deleteAndNotify( - any(), any(), anyString(), anyList(), anyBoolean(), any()); + any(), any(), any(AttributeScope.class), anyList(), anyBoolean(), any()); } @AfterEach From 5ecd9b623844093095a537e2c4ad380fbde5271c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 9 Jan 2024 14:47:01 +0200 Subject: [PATCH 12/17] fixed tests --- .../service/telemetry/DefaultTelemetrySubscriptionService.java | 1 - .../rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 531f518579..2424bd8676 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -243,7 +243,6 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public void saveAndNotify(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, FutureCallback callback) { saveAndNotify(tenantId, entityId, scope, attributes, true, callback); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java index f3fc7e6163..be25c71cee 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesNodeTest.java @@ -153,6 +153,6 @@ public class TbMsgDeleteAttributesNodeTest { } verify(ctx, times(1)).tellSuccess(newMsgCaptor.capture()); verify(ctx, never()).tellFailure(any(), any()); - verify(telemetryService, times(1)).deleteAndNotify(any(), any(), anyString(), anyList(), eq(notifyDevice || notifyDeviceMetadata), any()); + verify(telemetryService, times(1)).deleteAndNotify(any(), any(), any(AttributeScope.class), anyList(), eq(notifyDevice || notifyDeviceMetadata), any()); } } From e94a7e6cf74f23143e5e76deb9f014fe06536af9 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 9 Jan 2024 18:43:07 +0200 Subject: [PATCH 13/17] fixed test --- .../org/thingsboard/server/controller/WebsocketApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index 47ce021dae..59fa730d2a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -700,7 +700,7 @@ public class WebsocketApiTest extends AbstractControllerTest { private void sendAttributes(TenantId tenantId, EntityId entityId, TbAttributeSubscriptionScope scope, List attrData) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); - tsService.saveAndNotify(tenantId, entityId, scope.name(), attrData, new FutureCallback() { + tsService.saveAndNotify(tenantId, entityId, scope.getAttributeScope(), attrData, new FutureCallback() { @Override public void onSuccess(@Nullable Void result) { log.debug("sendAttributes callback onSuccess"); From 192951d4eb4302f915999b1f1db3701c6f4cd6eb Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 9 Jan 2024 19:10:59 +0200 Subject: [PATCH 14/17] minor refactoring --- .../server/dao/attributes/CachedAttributesService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index f87827b535..c5270cc830 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -144,8 +144,8 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { - return find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys); + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, final Collection attributeKeysNonUnique) { + return find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeysNonUnique); } @Override From 5b513b8d8e9a7f7def2aa8a5ce940cb4a9601a38 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 10 Jan 2024 12:06:42 +0200 Subject: [PATCH 15/17] deleted redundant method --- .../service/install/SqlDatabaseUpgradeService.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index cbd8686918..7045280f0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -819,18 +819,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService Thread.sleep(5000); } - private void executeQuery(Connection conn, String query) { - try { - Statement statement = conn.createStatement(); - statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script - printWarnings(statement); - log.info("Successfully executed query: {}", query); - } catch (SQLException e) { - log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); - throw new RuntimeException("Failed to execute query:" + query + " due to: ", e); - } - } - protected void printWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { From 44b5e91676e9b84112f80299c195b8b58327826c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 10 Jan 2024 15:56:24 +0200 Subject: [PATCH 16/17] updated upgrade script: inlined function, moved view dropping before table rename --- .../main/data/upgrade/3.6.3/schema_update.sql | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/application/src/main/data/upgrade/3.6.3/schema_update.sql b/application/src/main/data/upgrade/3.6.3/schema_update.sql index e11c52f77f..19e0d44839 100644 --- a/application/src/main/data/upgrade/3.6.3/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.3/schema_update.sql @@ -21,6 +21,8 @@ $$ BEGIN -- in case of running the upgrade script a second time: IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'attribute_kv' and column_name='entity_type') THEN + DROP VIEW IF EXISTS device_info_view; + DROP VIEW IF EXISTS device_info_active_attribute_view; ALTER INDEX IF EXISTS idx_attribute_kv_by_key_and_last_update_ts RENAME TO idx_attribute_kv_by_key_and_last_update_ts_old; IF EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'attribute_kv_pkey') THEN ALTER TABLE attribute_kv RENAME CONSTRAINT attribute_kv_pkey TO attribute_kv_pkey_old; @@ -40,8 +42,6 @@ $$ CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); END IF; - DROP VIEW IF EXISTS device_info_view; - DROP VIEW IF EXISTS device_info_active_attribute_view; END; $$; @@ -61,22 +61,6 @@ $$ END; $$; --- create to_attribute_type_id -CREATE OR REPLACE FUNCTION to_attribute_type_id(IN attribute_type varchar, OUT attribute_type_id int) AS -$$ -BEGIN - CASE - WHEN attribute_type = 'CLIENT_SCOPE' THEN - attribute_type_id := 1; - WHEN attribute_type = 'SERVER_SCOPE' THEN - attribute_type_id := 2; - WHEN attribute_type = 'SHARED_SCOPE' THEN - attribute_type_id := 3; - END CASE; -END; -$$ LANGUAGE plpgsql; - - -- insert keys into key_dictionary DO $$ @@ -96,7 +80,13 @@ DECLARE BEGIN IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'attribute_kv_old') THEN INSERT INTO attribute_kv(entity_id, attribute_type, attribute_key, bool_v, str_v, long_v, dbl_v, json_v, last_update_ts) - SELECT a.entity_id, to_attribute_type_id(a.attribute_type), k.key_id, a.bool_v, a.str_v, a.long_v, a.dbl_v, a.json_v, a.last_update_ts + SELECT a.entity_id, CASE + WHEN a.attribute_type = 'CLIENT_SCOPE' THEN 1 + WHEN a.attribute_type = 'SERVER_SCOPE' THEN 2 + WHEN a.attribute_type = 'SHARED_SCOPE' THEN 3 + ELSE 0 + END, + k.key_id, a.bool_v, a.str_v, a.long_v, a.dbl_v, a.json_v, a.last_update_ts FROM attribute_kv_old a INNER JOIN key_dictionary k ON (a.attribute_key = k.key) WHERE a.attribute_type IN ('SERVER_SCOPE', 'CLIENT_SCOPE', 'SHARED_SCOPE'); SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; From d77b735de39f13d8aacc2190cb60519fe6df269c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 10 Jan 2024 16:31:03 +0200 Subject: [PATCH 17/17] updated upgrade script: deleted redundant WHERE --- application/src/main/data/upgrade/3.6.3/schema_update.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/data/upgrade/3.6.3/schema_update.sql b/application/src/main/data/upgrade/3.6.3/schema_update.sql index 19e0d44839..9888ee0304 100644 --- a/application/src/main/data/upgrade/3.6.3/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.3/schema_update.sql @@ -87,8 +87,7 @@ BEGIN ELSE 0 END, k.key_id, a.bool_v, a.str_v, a.long_v, a.dbl_v, a.json_v, a.last_update_ts - FROM attribute_kv_old a INNER JOIN key_dictionary k ON (a.attribute_key = k.key) - WHERE a.attribute_type IN ('SERVER_SCOPE', 'CLIENT_SCOPE', 'SHARED_SCOPE'); + FROM attribute_kv_old a INNER JOIN key_dictionary k ON (a.attribute_key = k.key); SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; SELECT COUNT(*) INTO row_num FROM attribute_kv; RAISE NOTICE 'Migrated % of % rows', row_num, row_num_old;