From c92a7ecf3a5d18b2f56c14a984ebe8e2bf7cf31e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 2 Jan 2024 17:46:29 +0200 Subject: [PATCH] deleted attribute_kv_dictionary table (used ts_kv_dictionary instead), renamed ts_kv_dictionary to key_dictionary --- .../schema_update_attribute_kv.sql | 27 +++++++----- .../install/ThingsboardInstallService.java | 6 +-- .../install/SqlDatabaseUpgradeService.java | 6 +-- .../CassandraTsLatestToSqlMigrateService.java | 24 +++++------ .../model/sql/AttributeKvDictionaryEntry.java | 41 ------------------- ...ey.java => KeyDictionaryCompositeKey.java} | 2 +- ...ictionary.java => KeyDictionaryEntry.java} | 6 +-- .../AttributeKvDictionaryRepository.java | 27 ------------ .../dao/sql/attributes/JpaAttributeDao.java | 22 +++++----- .../dao/sql/query/EntityKeyMapping.java | 6 +-- .../sqlts/BaseAbstractSqlTimeseriesDao.java | 24 +++++------ ...tory.java => KeyDictionaryRepository.java} | 11 +++-- .../latest/SearchTsKvLatestRepository.java | 4 +- .../sqlts/latest/TsKvLatestRepository.java | 18 ++++---- .../main/resources/sql/schema-entities.sql | 10 +---- .../main/resources/sql/schema-timescale.sql | 6 +-- dao/src/main/resources/sql/schema-ts-psql.sql | 12 +++--- .../sql/schema-views-and-functions.sql | 8 ++-- .../profile/TbDeviceProfileNodeTest.java | 3 -- .../tools/migrator/DictionaryParser.java | 2 +- 20 files changed, 97 insertions(+), 168 deletions(-) rename application/src/main/data/upgrade/{3.6.2 => 3.6.3}/schema_update_attribute_kv.sql (88%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/{TsKvDictionaryCompositeKey.java => KeyDictionaryCompositeKey.java} (93%) rename dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/{TsKvDictionary.java => KeyDictionaryEntry.java} (91%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java rename dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/{TsKvDictionaryRepository.java => KeyDictionaryRepository.java} (65%) diff --git a/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql b/application/src/main/data/upgrade/3.6.3/schema_update_attribute_kv.sql similarity index 88% rename from application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql rename to application/src/main/data/upgrade/3.6.3/schema_update_attribute_kv.sql index 8a13029310..509a94bf20 100644 --- a/application/src/main/data/upgrade/3.6.2/schema_update_attribute_kv.sql +++ b/application/src/main/data/upgrade/3.6.3/schema_update_attribute_kv.sql @@ -46,13 +46,18 @@ $$ END; $$; --- create attribute_kv_dictionary table -CREATE TABLE IF NOT EXISTS attribute_kv_dictionary -( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT attribute_key_id_pkey PRIMARY KEY (key) -); +-- rename ts_kv_dictionary table to key_dictionary +DO +$$ + BEGIN + IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'ts_kv_dictionary') THEN + ALTER TABLE ts_kv_dictionary + RENAME CONSTRAINT ts_key_id_pkey TO key_id_pkey; + ALTER TABLE ts_kv_dictionary + RENAME TO key_dictionary; + END IF; + END; +$$; -- create to_attribute_type_id CREATE OR REPLACE FUNCTION to_attribute_type_id(IN attribute_type varchar, OUT attribute_type_id int) AS @@ -70,7 +75,7 @@ END; $$ LANGUAGE plpgsql; --- insert keys into attribute_kv_dictionary +-- insert keys into key_dictionary DO $$ DECLARE @@ -84,8 +89,8 @@ BEGIN LOOP FETCH key_cursor INTO insert_record; EXIT WHEN NOT FOUND; - IF NOT EXISTS(SELECT key FROM attribute_kv_dictionary WHERE key = insert_record.attribute_key) THEN - INSERT INTO attribute_kv_dictionary(key) VALUES (insert_record.attribute_key); + IF NOT EXISTS(SELECT key FROM key_dictionary WHERE key = insert_record.attribute_key) THEN + INSERT INTO key_dictionary(key) VALUES (insert_record.attribute_key); END IF; END LOOP; CLOSE key_cursor; @@ -122,7 +127,7 @@ BEGIN dbl_v, json_v, last_update_ts - FROM attribute_kv_old INNER JOIN attribute_kv_dictionary ON (attribute_kv_old.attribute_key = attribute_kv_dictionary.key) + FROM attribute_kv_old INNER JOIN key_dictionary ON (attribute_kv_old.attribute_key = key_dictionary.key) WHERE attribute_type= ANY(%L)) AS records) TO %L;', attribute_scope_array, path_to_file); EXECUTE format('COPY attribute_kv FROM %L', path_to_file); SELECT COUNT(*) INTO row_num_old FROM attribute_kv_old; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index f9fcf77001..383d22c21f 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -265,9 +265,9 @@ public class ThingsboardInstallService { case "3.6.0": log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); - case "3.6.2": - log.info("Upgrading ThingsBoard from version 3.6.2 to 3.7.0 ..."); - databaseEntitiesUpgradeService.upgradeDatabase("3.6.2"); + case "3.6.3": + log.info("Upgrading ThingsBoard from version 3.6.3 to 3.7.0 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.6.3"); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 1a22b6e0e2..1ad5c9afe4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -799,11 +799,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; - case "3.6.2": + case "3.6.3": try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - if (isOldSchema(conn, 3006002)) { + if (isOldSchema(conn, 3006003)) { log.info("Updating schema ..."); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.6.2", LOAD_ATTRIBUTE_KV_FUNCTIONS_SQL); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.6.3", LOAD_ATTRIBUTE_KV_FUNCTIONS_SQL); loadSql(schemaUpdateFile, conn); Path pathToTempAttributeKvFile; diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java index 4357e6be63..083d513e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java @@ -24,10 +24,10 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.dao.cassandra.CassandraCluster; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.util.NoSqlTsDao; import org.thingsboard.server.dao.util.SqlTsLatestDao; @@ -71,7 +71,7 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ protected CassandraCluster cluster; @Autowired - protected TsKvDictionaryRepository dictionaryRepository; + protected KeyDictionaryRepository dictionaryRepository; @Autowired private InstallScripts installScripts; @@ -192,22 +192,22 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ Integer keyId = tsKvDictionaryMap.get(strKey); if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (!tsKvDictionaryOptional.isPresent()) { tsCreationLock.lock(); try { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (!tsKvDictionaryOptional.isPresent()) { - TsKvDictionary tsKvDictionary = new TsKvDictionary(); - tsKvDictionary.setKey(strKey); + KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); + keyDictionaryEntry.setKey(strKey); try { - TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + KeyDictionaryEntry saved = dictionaryRepository.save(keyDictionaryEntry); tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); } catch (ConstraintViolationException e) { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); keyId = dictionary.getKeyId(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java deleted file mode 100644 index 98e3a0f5ab..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvDictionaryEntry.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.model.sql; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import lombok.Data; -import org.hibernate.annotations.Generated; - -import static org.thingsboard.server.dao.model.ModelConstants.KEY_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; - -@Data -@Entity -@Table(name = "attribute_kv_dictionary") -public final class AttributeKvDictionaryEntry { - - @Id - @Column(name = KEY_COLUMN) - private String key; - - @Column(name = KEY_ID_COLUMN, unique = true, columnDefinition="int") - @Generated - private int keyId; - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryCompositeKey.java similarity index 93% rename from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryCompositeKey.java index 49ab8b6822..599c0e5eb7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionaryCompositeKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryCompositeKey.java @@ -25,7 +25,7 @@ import java.io.Serializable; @Data @NoArgsConstructor @AllArgsConstructor -public class TsKvDictionaryCompositeKey implements Serializable{ +public class KeyDictionaryCompositeKey implements Serializable{ @Transient private static final long serialVersionUID = -4089175869616037523L; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryEntry.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java rename to dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryEntry.java index 5f1ff6a9c1..6063cbf70a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/TsKvDictionary.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/dictionary/KeyDictionaryEntry.java @@ -28,9 +28,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.KEY_ID_COLUMN; @Data @Entity -@Table(name = "ts_kv_dictionary") -@IdClass(TsKvDictionaryCompositeKey.class) -public final class TsKvDictionary { +@Table(name = "key_dictionary") +@IdClass(KeyDictionaryCompositeKey.class) +public final class KeyDictionaryEntry { @Id @Column(name = KEY_COLUMN) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java deleted file mode 100644 index aa0ca5b8a5..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvDictionaryRepository.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.attributes; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.dao.model.sql.AttributeKvDictionaryEntry; - -import java.util.Optional; - -public interface AttributeKvDictionaryRepository extends JpaRepository { - - Optional findByKeyId(int keyId); - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 50ee8c462c..01149d3d49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -36,12 +36,14 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; -import org.thingsboard.server.dao.model.sql.AttributeKvDictionaryEntry; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; @@ -64,7 +66,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl ScheduledLogExecutorComponent logExecutor; @Autowired - private AttributeKvDictionaryRepository dictionaryRepository; + private KeyDictionaryRepository keyDictionaryRepository; @Autowired private AttributeKvRepository attributeKvRepository; @@ -215,21 +217,21 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl private Integer getOrSaveKeyId(String attributeKey) { Integer keyId = attributeDictionaryMap.get(attributeKey); if (keyId == null) { - Optional byIdOptional = dictionaryRepository.findById(attributeKey); + Optional byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); if (byIdOptional.isEmpty()) { attributeCreationLock.lock(); try { - byIdOptional = dictionaryRepository.findById(attributeKey); + byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); if (byIdOptional.isEmpty()) { - AttributeKvDictionaryEntry attributeKvDictionaryEntry = new AttributeKvDictionaryEntry(); + KeyDictionaryEntry attributeKvDictionaryEntry = new KeyDictionaryEntry(); attributeKvDictionaryEntry.setKey(attributeKey); try { - AttributeKvDictionaryEntry saved = dictionaryRepository.save(attributeKvDictionaryEntry); + KeyDictionaryEntry saved = keyDictionaryRepository.save(attributeKvDictionaryEntry); attributeDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); } catch (DataIntegrityViolationException | ConstraintViolationException e) { - byIdOptional = dictionaryRepository.findById(attributeKey); - AttributeKvDictionaryEntry dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); + byIdOptional = keyDictionaryRepository.findById(new KeyDictionaryCompositeKey(attributeKey)); + KeyDictionaryEntry dictionary = byIdOptional.orElseThrow(() -> new RuntimeException("Failed to get AttributeKvDictionary entity from DB!")); attributeDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); keyId = dictionary.getKeyId(); } @@ -247,7 +249,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return keyId; } private String getKey(Integer attributeKey) { - Optional byKeyId = dictionaryRepository.findByKeyId(attributeKey); - return byKeyId.map(AttributeKvDictionaryEntry::getKey).orElse(null); + Optional byKeyId = keyDictionaryRepository.findByKeyId(attributeKey); + return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 200e93765d..23d8e972b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -256,13 +256,13 @@ public class EntityKeyMapping { } if (entityKey.getType().equals(EntityKeyType.TIME_SERIES)) { String join = (hasFilter() && hasFilterValues(ctx)) ? "inner join" : "left join"; - return String.format("%s ts_kv_latest %s ON %s.entity_id=entities.id AND %s.key = (select key_id from ts_kv_dictionary where key = :%s_key_id) %s", + return String.format("%s ts_kv_latest %s ON %s.entity_id=entities.id AND %s.key = (select key_id from key_dictionary where key = :%s_key_id) %s", join, alias, alias, alias, alias, filterQuery); } else { String query; if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) { String join = (hasFilter() && hasFilterValues(ctx)) ? "inner join" : "left join"; - query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.attribute_key=(select key_id from attribute_kv_dictionary where key = :%s_key_id) ", + query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.attribute_key=(select key_id from key_dictionary where key = :%s_key_id) ", join, alias, alias, alias, alias); int scope; if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { @@ -275,7 +275,7 @@ public class EntityKeyMapping { query = String.format("%s AND %s.attribute_type=%s %s", query, alias, scope, filterQuery); } else { String join = (hasFilter() && hasFilterValues(ctx)) ? "join LATERAL" : "left join LATERAL"; - query = String.format("%s (select * from attribute_kv %s WHERE %s.entity_id=entities.id AND %s.attribute_key=(select key_id from attribute_kv_dictionary where key = :%s_key_id) %s " + + query = String.format("%s (select * from attribute_kv %s WHERE %s.entity_id=entities.id AND %s.attribute_key=(select key_id from key_dictionary where key = :%s_key_id) %s " + "ORDER BY %s.last_update_ts DESC limit 1) as %s ON true", join, alias, alias, alias, alias, filterQuery, alias, alias); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 52c4216c69..9a33cba81e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -26,10 +26,10 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import org.thingsboard.server.dao.sqlts.dictionary.KeyDictionaryRepository; import jakarta.annotation.Nullable; import java.util.List; @@ -46,13 +46,13 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); protected static final ReentrantLock tsCreationLock = new ReentrantLock(); @Autowired - protected TsKvDictionaryRepository dictionaryRepository; + protected KeyDictionaryRepository dictionaryRepository; protected Integer getOrSaveKeyId(String strKey) { Integer keyId = tsKvDictionaryMap.get(strKey); if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (tsKvDictionaryOptional.isEmpty()) { tsCreationLock.lock(); try { @@ -60,17 +60,17 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni if (keyId != null) { return keyId; } - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); if (tsKvDictionaryOptional.isEmpty()) { - TsKvDictionary tsKvDictionary = new TsKvDictionary(); - tsKvDictionary.setKey(strKey); + KeyDictionaryEntry keyDictionaryEntry = new KeyDictionaryEntry(); + keyDictionaryEntry.setKey(strKey); try { - TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + KeyDictionaryEntry saved = dictionaryRepository.save(keyDictionaryEntry); tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); } catch (DataIntegrityViolationException | ConstraintViolationException e) { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryOptional = dictionaryRepository.findById(new KeyDictionaryCompositeKey(strKey)); + KeyDictionaryEntry dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); keyId = dictionary.getKeyId(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java similarity index 65% rename from dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java index 41e165b92f..12c2422094 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java @@ -16,15 +16,14 @@ package org.thingsboard.server.dao.sqlts.dictionary; import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; -import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import java.util.Optional; -@SqlTsOrTsLatestAnyDao -public interface TsKvDictionaryRepository extends JpaRepository { +public interface KeyDictionaryRepository extends JpaRepository { + + Optional findByKeyId(int keyId); - Optional findByKeyId(int keyId); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java index beaa4a5f0d..fc7b2d568f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java @@ -30,9 +30,9 @@ public class SearchTsKvLatestRepository { public static final String FIND_ALL_BY_ENTITY_ID = "findAllByEntityId"; - public static final String FIND_ALL_BY_ENTITY_ID_QUERY = "SELECT ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key, ts_kv_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue," + + public static final String FIND_ALL_BY_ENTITY_ID_QUERY = "SELECT ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key, key_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue," + " ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue, ts_kv_latest.ts AS ts FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)"; + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)"; @PersistenceContext private EntityManager entityManager; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 835ce3a11b..abcae06e19 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -26,19 +26,19 @@ import java.util.UUID; public interface TsKvLatestRepository extends JpaRepository { - @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " + - "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE device_profile_id = :device_profile_id AND tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true) + @Query(value = "SELECT DISTINCT key_dictionary.key AS strKey FROM ts_kv_latest " + + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id " + + "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE device_profile_id = :device_profile_id AND tenant_id = :tenant_id limit 100) ORDER BY key_dictionary.key", nativeQuery = true) List getKeysByDeviceProfileId(@Param("tenant_id") UUID tenantId, @Param("device_profile_id") UUID deviceProfileId); - @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " + - "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE tenant_id = :tenant_id limit 100) ORDER BY ts_kv_dictionary.key", nativeQuery = true) + @Query(value = "SELECT DISTINCT key_dictionary.key AS strKey FROM ts_kv_latest " + + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id " + + "WHERE ts_kv_latest.entity_id IN (SELECT id FROM device WHERE tenant_id = :tenant_id limit 100) ORDER BY key_dictionary.key", nativeQuery = true) List getKeysByTenantId(@Param("tenant_id") UUID tenantId); - @Query(value = "SELECT DISTINCT ts_kv_dictionary.key AS strKey FROM ts_kv_latest " + - "INNER JOIN ts_kv_dictionary ON ts_kv_latest.key = ts_kv_dictionary.key_id " + - "WHERE ts_kv_latest.entity_id IN :entityIds ORDER BY ts_kv_dictionary.key", nativeQuery = true) + @Query(value = "SELECT DISTINCT key_dictionary.key AS strKey FROM ts_kv_latest " + + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id " + + "WHERE ts_kv_latest.entity_id IN :entityIds ORDER BY key_dictionary.key", nativeQuery = true) List findAllKeysByEntityIds(@Param("entityIds") List entityIds); } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index f376a777d3..9ce78bd56f 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -115,12 +115,6 @@ CREATE TABLE IF NOT EXISTS attribute_kv ( CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); -CREATE TABLE IF NOT EXISTS attribute_kv_dictionary ( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT attribute_key_id_pkey PRIMARY KEY (key) -); - CREATE TABLE IF NOT EXISTS component_descriptor ( id uuid NOT NULL CONSTRAINT component_descriptor_pkey PRIMARY KEY, created_time bigint NOT NULL, @@ -552,11 +546,11 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); -CREATE TABLE IF NOT EXISTS ts_kv_dictionary +CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, - CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) + CONSTRAINT key_id_pkey PRIMARY KEY (key) ); CREATE TABLE IF NOT EXISTS oauth2_params ( diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 0a60b64a50..f547a72cd1 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS ts_kv ( CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ); -CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( +CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) @@ -104,7 +104,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -122,7 +122,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 3f8f380b03..023e1b544d 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS ts_kv CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ) PARTITION BY RANGE (ts); -CREATE TABLE IF NOT EXISTS ts_kv_dictionary +CREATE TABLE IF NOT EXISTS key_dictionary ( key varchar(255) NOT NULL, key_id serial UNIQUE, @@ -75,7 +75,7 @@ BEGIN WHERE schemaname = 'public' AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' + AND tablename != 'key_dictionary' AND tablename != 'ts_kv_indefinite' AND tablename != partition_by_max_ttl_date LOOP @@ -96,7 +96,7 @@ BEGIN WHERE schemaname = 'public' AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' + AND tablename != 'key_dictionary' AND tablename != 'ts_kv_indefinite' AND tablename != partition_by_max_ttl_date LOOP @@ -138,7 +138,7 @@ BEGIN WHERE schemaname = 'public' AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' + AND tablename != 'key_dictionary' AND tablename != 'ts_kv_indefinite' AND tablename != partition_by_max_ttl_date LOOP @@ -272,7 +272,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -290,7 +290,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/dao/src/main/resources/sql/schema-views-and-functions.sql b/dao/src/main/resources/sql/schema-views-and-functions.sql index 4d684c6406..bcb34a030a 100644 --- a/dao/src/main/resources/sql/schema-views-and-functions.sql +++ b/dao/src/main/resources/sql/schema-views-and-functions.sql @@ -23,7 +23,7 @@ SELECT d.* , COALESCE(da.bool_v, FALSE) as active FROM device d LEFT JOIN customer c ON c.id = d.customer_id - LEFT JOIN attribute_kv da ON da.entity_id = d.id AND da.attribute_type = 2 AND da.attribute_key = (select key_id from attribute_kv_dictionary where key = 'active'); + LEFT JOIN attribute_kv da ON da.entity_id = d.id AND da.attribute_type = 2 AND da.attribute_key = (select key_id from key_dictionary where key = 'active'); DROP VIEW IF EXISTS device_info_active_ts_view CASCADE; CREATE OR REPLACE VIEW device_info_active_ts_view AS @@ -34,7 +34,7 @@ SELECT d.* , COALESCE(dt.bool_v, FALSE) as active FROM device d LEFT JOIN customer c ON c.id = d.customer_id - LEFT JOIN ts_kv_latest dt ON dt.entity_id = d.id and dt.key = (select key_id from ts_kv_dictionary where key = 'active'); + LEFT JOIN ts_kv_latest dt ON dt.entity_id = d.id and dt.key = (select key_id from key_dictionary where key = 'active'); DROP VIEW IF EXISTS device_info_view CASCADE; CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view; @@ -312,7 +312,7 @@ BEGIN WHILE FOUND LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', tenant_id_record, 'TTL') INTO tenant_ttl; if tenant_ttl IS NULL THEN tenant_ttl := system_ttl; @@ -330,7 +330,7 @@ BEGIN SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record LOOP EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from attribute_kv_dictionary where key = %L)', + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = (select key_id from key_dictionary where key = %L)', customer_id_record, 'TTL') INTO customer_ttl; IF customer_ttl IS NULL THEN customer_ttl_ts := tenant_ttl_ts; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 0b94f07255..2aafcc0381 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -30,10 +30,8 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; import org.thingsboard.server.common.data.alarm.AlarmInfo; @@ -60,7 +58,6 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.DynamicValueSourceType; -import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.NumericFilterPredicate; diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java b/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java index 3b1164e95b..b85171d6ac 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java @@ -40,7 +40,7 @@ public class DictionaryParser { } private boolean isBlockStarted(String line) { - return line.startsWith("COPY public.ts_kv_dictionary ("); + return line.startsWith("COPY public.key_dictionary ("); } private void parseDictionaryDump(LineIterator iterator) throws IOException {