From 9b54f009e6a9c6e2b01714001c9b39a009ef7c20 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 7 Nov 2022 11:29:27 +0100 Subject: [PATCH 1/7] upgrade asset-profiles using ExecutorService --- .../install/DbUpgradeExecutorService.java | 26 ++++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 30 +++++++++++-------- 2 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java diff --git a/application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java b/application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java new file mode 100644 index 0000000000..b3ce77ac7a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; +import org.thingsboard.server.service.executors.DbCallbackExecutorService; + +@Component +@Profile("install") +public class DbUpgradeExecutorService extends DbCallbackExecutorService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 092726d0c8..668f20ce63 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.install; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -33,13 +35,11 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.rule.RuleChainService; -import org.thingsboard.server.dao.tenant.TenantProfileService; +import org.thingsboard.server.dao.sql.asset.AssetRepository; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -56,7 +56,10 @@ import java.sql.SQLException; import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; @@ -110,7 +113,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService private DeviceService deviceService; @Autowired - private AssetService assetService; + private AssetRepository assetRepository; @Autowired private DeviceProfileService deviceProfileService; @@ -129,10 +132,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService private TbRuleEngineQueueConfigService queueConfig; @Autowired - private RuleChainService ruleChainService; - - @Autowired - private TenantProfileService tenantProfileService; + private DbUpgradeExecutorService dbUpgradeExecutor; @Override public void upgradeDatabase(String fromVersion) throws Exception { @@ -623,17 +623,20 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.info("Creating default asset profiles..."); PageLink pageLink = new PageLink(100); PageData pageData; + List> futures = new ArrayList<>(); do { pageData = tenantService.findTenants(pageLink); for (Tenant tenant : pageData.getData()) { - List assetTypes = assetService.findAssetTypesByTenantId(tenant.getId()).get(); + Set assetTypes = new HashSet<>(assetRepository.findTenantAssetTypes(tenant.getUuidId())); + assetTypes.remove("default"); + try { - assetProfileService.createDefaultAssetProfile(tenant.getId()); + futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.createDefaultAssetProfile(tenant.getId()))); } catch (Exception e) { } - for (EntitySubtype assetType : assetTypes) { + for (String assetType : assetTypes) { try { - assetProfileService.findOrCreateAssetProfile(tenant.getId(), assetType.getType()); + futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.findOrCreateAssetProfile(tenant.getId(), assetType))); } catch (Exception e) { } } @@ -641,6 +644,8 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService pageLink = pageLink.nextPageLink(); } while (pageData.hasNext()); + Futures.allAsList(futures).get(); + log.info("Updating asset profiles..."); conn.createStatement().execute("call update_asset_profiles()"); @@ -728,5 +733,4 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService return queue; } - } From a9dabe14ca191e2d0a92adf210d70d8bca409555 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 7 Nov 2022 11:51:09 +0100 Subject: [PATCH 2/7] refactored --- .../server/service/install/SqlDatabaseUpgradeService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 668f20ce63..4b6f1e1a52 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -57,9 +57,7 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; @@ -627,7 +625,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService do { pageData = tenantService.findTenants(pageLink); for (Tenant tenant : pageData.getData()) { - Set assetTypes = new HashSet<>(assetRepository.findTenantAssetTypes(tenant.getUuidId())); + List assetTypes = assetRepository.findTenantAssetTypes(tenant.getUuidId()); assetTypes.remove("default"); try { From 2d2d78ae6bc656d9ed36375743cfcf5455e9e7b7 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 8 Nov 2022 15:08:22 +0100 Subject: [PATCH 3/7] fetch attribute types for upgrade improvements --- .../install/SqlDatabaseUpgradeService.java | 39 +++++++++++-------- .../server/dao/asset/AssetDao.java | 3 ++ .../server/dao/model/sql/TbPair.java | 26 +++++++++++++ .../server/dao/sql/asset/AssetRepository.java | 18 +++++---- .../server/dao/sql/asset/JpaAssetDao.java | 9 +++++ 5 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 4b6f1e1a52..3067e545b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -34,12 +34,13 @@ import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.model.sql.TbPair; import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.sql.asset.AssetRepository; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -57,7 +58,10 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; @@ -111,7 +115,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService private DeviceService deviceService; @Autowired - private AssetRepository assetRepository; + private AssetDao assetDao; @Autowired private DeviceProfileService deviceProfileService; @@ -619,24 +623,27 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService loadSql(schemaUpdateFile, conn); log.info("Creating default asset profiles..."); - PageLink pageLink = new PageLink(100); - PageData pageData; + PageLink pageLink = new PageLink(1000); + PageData> pageData; List> futures = new ArrayList<>(); + Set tenants = new HashSet<>(); do { - pageData = tenantService.findTenants(pageLink); - for (Tenant tenant : pageData.getData()) { - List assetTypes = assetRepository.findTenantAssetTypes(tenant.getUuidId()); - assetTypes.remove("default"); - - try { - futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.createDefaultAssetProfile(tenant.getId()))); - } catch (Exception e) { + pageData = assetDao.getAllAssetTypes(pageLink); + for (TbPair pair : pageData.getData()) { + TenantId tenantId = new TenantId(pair.getFirst()); + String assetType = pair.getSecond(); + if (tenants.add(tenantId)) { + try { + futures.add(dbUpgradeExecutor.submit(() -> + assetProfileService.createDefaultAssetProfile(tenantId))); + } catch (Exception e) {} } - for (String assetType : assetTypes) { + + if (!"default".equals(assetType)) { try { - futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.findOrCreateAssetProfile(tenant.getId(), assetType))); - } catch (Exception e) { - } + futures.add(dbUpgradeExecutor.submit(() -> + assetProfileService.findOrCreateAssetProfile(tenantId, assetType))); + } catch (Exception e) {} } } pageLink = pageLink.nextPageLink(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 4d67a794b6..d17181cffa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.model.sql.TbPair; import java.util.List; import java.util.Optional; @@ -222,4 +223,6 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD * @return the list of asset objects */ PageData findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink); + + PageData> getAllAssetTypes(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java new file mode 100644 index 0000000000..4cea485575 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class TbPair { + private S first; + private T second; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index eadf0d2b84..cbb68b6ccc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -23,6 +23,7 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; +import org.thingsboard.server.dao.model.sql.TbPair; import java.util.List; import java.util.UUID; @@ -70,9 +71,9 @@ public interface AssetRepository extends JpaRepository, Expor "AND a.assetProfileId = :profileId " + "AND LOWER(a.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") Page findByTenantIdAndProfileId(@Param("tenantId") UUID tenantId, - @Param("profileId") UUID profileId, - @Param("searchText") String searchText, - Pageable pageable); + @Param("profileId") UUID profileId, + @Param("searchText") String searchText, + Pageable pageable); @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo, p.name) " + "FROM AssetEntity a " + @@ -186,14 +187,17 @@ public interface AssetRepository extends JpaRepository, Expor "AND a.type = :type " + "AND LOWER(a.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") Page findByTenantIdAndEdgeIdAndType(@Param("tenantId") UUID tenantId, - @Param("edgeId") UUID edgeId, - @Param("type") String type, - @Param("searchText") String searchText, - Pageable pageable); + @Param("edgeId") UUID edgeId, + @Param("type") String type, + @Param("searchText") String searchText, + Pageable pageable); Long countByTenantIdAndTypeIsNot(UUID tenantId, String type); @Query("SELECT externalId FROM AssetEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); + @Query(value = "SELECT DISTINCT new org.thingsboard.server.dao.model.sql.TbPair(a.tenantId , a.type) FROM AssetEntity a") + Page> getAllAssetTypes(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 489e15502e..a0c70428a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -28,14 +28,17 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; +import org.thingsboard.server.dao.model.sql.TbPair; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -243,6 +246,12 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im DaoUtil.toPageable(pageLink))); } + public PageData> getAllAssetTypes(PageLink pageLink) { + log.debug("Try to find all asset types and pageLink [{}]", pageLink); + return DaoUtil.pageToPageData(assetRepository.getAllAssetTypes( + DaoUtil.toPageable(pageLink, Arrays.asList(new SortOrder("tenantId"), new SortOrder("type"))))); + } + @Override public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE); From 0849aa722f8f3b310d57a1be3db233f8dee479fc Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 8 Nov 2022 17:34:44 +0100 Subject: [PATCH 4/7] delete assets with a non-existent tenant and set default asset profile to the assets with xss --- .../server/service/install/SqlDatabaseUpgradeService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 3067e545b1..777284521e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -622,6 +622,8 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.1", "schema_update_before.sql"); loadSql(schemaUpdateFile, conn); + conn.createStatement().execute("DELETE FROM asset a WHERE NOT exists(SELECT id FROM tenant WHERE id = a.tenant_id);"); + log.info("Creating default asset profiles..."); PageLink pageLink = new PageLink(1000); PageData> pageData; @@ -654,6 +656,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.info("Updating asset profiles..."); conn.createStatement().execute("call update_asset_profiles()"); + conn.createStatement().execute("UPDATE asset a SET asset_profile_id = " + + "(SELECT id FROM asset_profile ap WHERE ap.tenant_id = a.tenant_id AND name='default') WHERE a.asset_profile_id IS NULL;"); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.1", "schema_update_after.sql"); loadSql(schemaUpdateFile, conn); From 71d26ba18b83edd9b17a5f5b9cf0a8a9df78886d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 9 Nov 2022 01:18:01 +0100 Subject: [PATCH 5/7] asset profile upgrade improvements --- .../install/SqlDatabaseUpgradeService.java | 36 +++++++++++++------ .../dao/sql/tenant/TenantRepository.java | 4 +++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 777284521e..bcd3686183 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.model.sql.TbPair; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.sql.tenant.TenantRepository; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -111,6 +112,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @Autowired private TenantService tenantService; + @Autowired + private TenantRepository tenantRepository; + @Autowired private DeviceService deviceService; @@ -628,29 +632,41 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService PageLink pageLink = new PageLink(1000); PageData> pageData; List> futures = new ArrayList<>(); - Set tenants = new HashSet<>(); + Set tenants = new HashSet<>(); do { pageData = assetDao.getAllAssetTypes(pageLink); for (TbPair pair : pageData.getData()) { TenantId tenantId = new TenantId(pair.getFirst()); String assetType = pair.getSecond(); - if (tenants.add(tenantId)) { - try { - futures.add(dbUpgradeExecutor.submit(() -> - assetProfileService.createDefaultAssetProfile(tenantId))); - } catch (Exception e) {} + if (tenants.add(pair.getFirst())) { + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.createDefaultAssetProfile(tenantId); + } catch (Exception e) {} + })); } if (!"default".equals(assetType)) { - try { - futures.add(dbUpgradeExecutor.submit(() -> - assetProfileService.findOrCreateAssetProfile(tenantId, assetType))); - } catch (Exception e) {} + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.findOrCreateAssetProfile(tenantId, assetType); + } catch (Exception e) {} + })); } } pageLink = pageLink.nextPageLink(); } while (pageData.hasNext()); + List tenantsWithoutProfiles = tenantRepository.getIdsNotIn(tenants); + + tenantsWithoutProfiles.forEach(uuid -> + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.createDefaultAssetProfile(TenantId.fromUUID(uuid)); + } catch (Exception e) {} + })) + ); + Futures.allAsList(futures).get(); log.info("Updating asset profiles..."); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 7042dcbb16..fb563895d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -24,6 +24,7 @@ import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; import java.util.List; +import java.util.Set; import java.util.UUID; /** @@ -54,4 +55,7 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); + @Query("SELECT t.id FROM TenantEntity t WHERE t.id NOT IN :ids") + List getIdsNotIn(@Param("ids")Set ids); + } From d84fd92f571b93b3d789421e7a0fb3aa2aec75b4 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 9 Nov 2022 09:58:15 +0100 Subject: [PATCH 6/7] refactored --- .../install/SqlDatabaseUpgradeService.java | 43 +++++++++---------- .../dao/sql/tenant/TenantRepository.java | 5 --- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index bcd3686183..d96475ad31 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -629,23 +629,30 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService conn.createStatement().execute("DELETE FROM asset a WHERE NOT exists(SELECT id FROM tenant WHERE id = a.tenant_id);"); log.info("Creating default asset profiles..."); - PageLink pageLink = new PageLink(1000); - PageData> pageData; List> futures = new ArrayList<>(); + + PageLink pageLink = new PageLink(1000); + PageData tenantIds; + do { + tenantIds = tenantService.findTenantsIds(pageLink); + for (TenantId tenantId : tenantIds.getData()) { + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.createDefaultAssetProfile(tenantId); + } catch (Exception e) {} + })); + } + pageLink = pageLink.nextPageLink(); + } while (tenantIds.hasNext()); + + pageLink = new PageLink(1000); + PageData> pairs; Set tenants = new HashSet<>(); do { - pageData = assetDao.getAllAssetTypes(pageLink); - for (TbPair pair : pageData.getData()) { + pairs = assetDao.getAllAssetTypes(pageLink); + for (TbPair pair : pairs.getData()) { TenantId tenantId = new TenantId(pair.getFirst()); String assetType = pair.getSecond(); - if (tenants.add(pair.getFirst())) { - futures.add(dbUpgradeExecutor.submit(() -> { - try { - assetProfileService.createDefaultAssetProfile(tenantId); - } catch (Exception e) {} - })); - } - if (!"default".equals(assetType)) { futures.add(dbUpgradeExecutor.submit(() -> { try { @@ -655,17 +662,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } pageLink = pageLink.nextPageLink(); - } while (pageData.hasNext()); - - List tenantsWithoutProfiles = tenantRepository.getIdsNotIn(tenants); - - tenantsWithoutProfiles.forEach(uuid -> - futures.add(dbUpgradeExecutor.submit(() -> { - try { - assetProfileService.createDefaultAssetProfile(TenantId.fromUUID(uuid)); - } catch (Exception e) {} - })) - ); + } while (pairs.hasNext()); Futures.allAsList(futures).get(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index fb563895d5..ac02d77bc8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -24,7 +24,6 @@ import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; import java.util.List; -import java.util.Set; import java.util.UUID; /** @@ -54,8 +53,4 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); - - @Query("SELECT t.id FROM TenantEntity t WHERE t.id NOT IN :ids") - List getIdsNotIn(@Param("ids")Set ids); - } From 03aafd4c3ca0065d6244ca473cbc9ed63b3b9bad Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 9 Nov 2022 13:28:42 +0100 Subject: [PATCH 7/7] added TbServiceQueue asset validation and upgrade improvements --- .../data/upgrade/3.4.1/schema_update_before.sql | 9 +++++---- .../server/controller/AssetController.java | 5 ----- .../entitiy/asset/DefaultTbAssetService.java | 14 ++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 15 +++++---------- .../server/common/data/util}/TbPair.java | 2 +- .../thingsboard/server/dao/asset/AssetDao.java | 2 +- .../server/dao/sql/asset/AssetRepository.java | 4 ++-- .../server/dao/sql/asset/JpaAssetDao.java | 2 +- 8 files changed, 29 insertions(+), 24 deletions(-) rename {dao/src/main/java/org/thingsboard/server/dao/model/sql => common/data/src/main/java/org/thingsboard/server/common/data/util}/TbPair.java (94%) diff --git a/application/src/main/data/upgrade/3.4.1/schema_update_before.sql b/application/src/main/data/upgrade/3.4.1/schema_update_before.sql index 59566e42b5..27f772aba3 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update_before.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update_before.sql @@ -37,9 +37,10 @@ CREATE OR REPLACE PROCEDURE update_asset_profiles() LANGUAGE plpgsql AS $$ BEGIN -UPDATE asset as a SET asset_profile_id = p.id - FROM - (SELECT id, tenant_id, name from asset_profile) as p -WHERE a.asset_profile_id IS NULL AND p.tenant_id = a.tenant_id AND a.type = p.name; + UPDATE asset a SET asset_profile_id = COALESCE( + (SELECT id from asset_profile p WHERE p.tenant_id = a.tenant_id AND a.type = p.name), + (SELECT id from asset_profile p WHERE p.tenant_id = a.tenant_id AND p.name = 'default') + ) + WHERE a.asset_profile_id IS NULL; END; $$; diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 8450c67ea3..968829b65a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -86,7 +85,6 @@ import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHO import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; import static org.thingsboard.server.controller.EdgeController.EDGE_ID; -import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; @RestController @TbCoreComponent @@ -148,9 +146,6 @@ public class AssetController extends BaseController { @RequestMapping(value = "/asset", method = RequestMethod.POST) @ResponseBody public Asset saveAsset(@ApiParam(value = "A JSON value representing the asset.") @RequestBody Asset asset) throws Exception { - if (TB_SERVICE_QUEUE.equals(asset.getType())) { - throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } asset.setTenantId(getTenantId()); checkEntity(asset.getId(), asset, Resource.ASSET); return tbAssetService.save(asset, getCurrentUser()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index 118dfe3b58..abaef626b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -22,8 +22,10 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; @@ -31,20 +33,32 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; import java.util.List; +import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; + @Service @AllArgsConstructor public class DefaultTbAssetService extends AbstractTbEntityService implements TbAssetService { private final AssetService assetService; + private final TbAssetProfileCache assetProfileCache; @Override public Asset save(Asset asset, User user) throws Exception { ActionType actionType = asset.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = asset.getTenantId(); try { + if (TB_SERVICE_QUEUE.equals(asset.getType())) { + throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } else if (asset.getAssetProfileId() != null) { + AssetProfile assetProfile = assetProfileCache.get(tenantId, asset.getAssetProfileId()); + if (assetProfile != null && TB_SERVICE_QUEUE.equals(assetProfile.getName())) { + throw new ThingsboardException("Unable to save asset with profile " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + } Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); autoCommit(user, savedAsset.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), savedAsset, diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index d96475ad31..80665d544f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -39,7 +39,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.sql.tenant.TenantRepository; import org.thingsboard.server.dao.tenant.TenantService; @@ -59,9 +59,7 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -629,11 +627,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService conn.createStatement().execute("DELETE FROM asset a WHERE NOT exists(SELECT id FROM tenant WHERE id = a.tenant_id);"); log.info("Creating default asset profiles..."); - List> futures = new ArrayList<>(); PageLink pageLink = new PageLink(1000); PageData tenantIds; do { + List> futures = new ArrayList<>(); tenantIds = tenantService.findTenantsIds(pageLink); for (TenantId tenantId : tenantIds.getData()) { futures.add(dbUpgradeExecutor.submit(() -> { @@ -642,13 +640,14 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } catch (Exception e) {} })); } + Futures.allAsList(futures).get(); pageLink = pageLink.nextPageLink(); } while (tenantIds.hasNext()); pageLink = new PageLink(1000); PageData> pairs; - Set tenants = new HashSet<>(); do { + List> futures = new ArrayList<>(); pairs = assetDao.getAllAssetTypes(pageLink); for (TbPair pair : pairs.getData()) { TenantId tenantId = new TenantId(pair.getFirst()); @@ -661,17 +660,13 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService })); } } + Futures.allAsList(futures).get(); pageLink = pageLink.nextPageLink(); } while (pairs.hasNext()); - Futures.allAsList(futures).get(); - log.info("Updating asset profiles..."); conn.createStatement().execute("call update_asset_profiles()"); - conn.createStatement().execute("UPDATE asset a SET asset_profile_id = " + - "(SELECT id FROM asset_profile ap WHERE ap.tenant_id = a.tenant_id AND name='default') WHERE a.asset_profile_id IS NULL;"); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.1", "schema_update_after.sql"); loadSql(schemaUpdateFile, conn); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java rename to common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java index 4cea485575..c7dba0c848 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.model.sql; +package org.thingsboard.server.common.data.util; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index d17181cffa..46a1a1bac5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import java.util.List; import java.util.Optional; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index cbb68b6ccc..aec62f0021 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -23,7 +23,7 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import java.util.List; import java.util.UUID; @@ -197,7 +197,7 @@ public interface AssetRepository extends JpaRepository, Expor @Query("SELECT externalId FROM AssetEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); - @Query(value = "SELECT DISTINCT new org.thingsboard.server.dao.model.sql.TbPair(a.tenantId , a.type) FROM AssetEntity a") + @Query(value = "SELECT DISTINCT new org.thingsboard.server.common.data.util.TbPair(a.tenantId , a.type) FROM AssetEntity a") Page> getAllAssetTypes(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index a0c70428a5..b0e1921e87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -33,7 +33,7 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao;