From c40fbf7dd9f46267aec504a66334c7ccf6779282 Mon Sep 17 00:00:00 2001 From: rusikv Date: Tue, 7 Nov 2023 18:44:46 +0200 Subject: [PATCH 001/410] UI: alarm table widget clear selection on entering edit mode --- .../main/data/json/system/widget_types/alarms_table.json | 2 +- .../widget/lib/alarm/alarms-table-widget.component.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/application/src/main/data/json/system/widget_types/alarms_table.json b/application/src/main/data/json/system/widget_types/alarms_table.json index 5a360bcc32..12716482c9 100644 --- a/application/src/main/data/json/system/widget_types/alarms_table.json +++ b/application/src/main/data/json/system/widget_types/alarms_table.json @@ -11,7 +11,7 @@ "resources": [], "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", + "controllerScript": "self.onInit = function() {\n}\n\nself.onDataUpdated = function() {\n self.ctx.$scope.alarmsTableWidget.onDataUpdated();\n}\n\nself.onEditModeChanged = function() {\n self.ctx.$scope.alarmsTableWidget.onEditModeChanged();\n}\n\nself.actionSources = function() {\n return {\n 'actionCellButton': {\n name: 'widget-action.action-cell-button',\n multiple: true,\n hasShowCondition: true\n },\n 'rowClick': {\n name: 'widget-action.row-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n}\n", "settingsSchema": "", "dataKeySettingsSchema": "", "settingsDirective": "tb-alarms-table-widget-settings", diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 20f123972a..133f484c28 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -321,6 +321,12 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.ctx.detectChanges(); } + public onEditModeChanged() { + if (this.alarmsDatasource.selection.hasValue()) { + this.alarmsDatasource.clearSelection(); + } + } + public pageLinkSortDirection(): SortDirection { return entityDataPageLinkSortDirection(this.pageLink); } From b75704e6feb3c44fcbd08f2c494398f731ff1682 Mon Sep 17 00:00:00 2001 From: rusikv Date: Wed, 8 Nov 2023 15:50:24 +0200 Subject: [PATCH 002/410] UI: alarm table widget on edit mode if selection has value do not hide title panel with edit action --- .../widget/lib/alarm/alarms-table-widget.component.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 133f484c28..20526591ef 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -322,8 +322,9 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, } public onEditModeChanged() { - if (this.alarmsDatasource.selection.hasValue()) { - this.alarmsDatasource.clearSelection(); + if (this.enableSelection && this.alarmsDatasource.selection.hasValue()) { + this.ctx.hideTitlePanel = !this.ctx.isEdit; + this.ctx.detectChanges(true); } } From acc2b068f514cbe99f16f82cb6e453d20c408f6d Mon Sep 17 00:00:00 2001 From: rusikv Date: Mon, 13 Nov 2023 17:50:13 +0200 Subject: [PATCH 003/410] Added ability to display right layout first in mobile view --- .../dashboard-page.component.html | 4 +- .../dashboard-page.component.ts | 7 ++++ .../dashboard-settings-dialog.component.html | 39 +++++++++++-------- .../dashboard-settings-dialog.component.ts | 7 ++++ ...nage-dashboard-layouts-dialog.component.ts | 3 +- .../src/app/shared/models/dashboard.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 1 + 7 files changed, 42 insertions(+), 20 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index a388f69252..1cbdd2fcf8 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -296,7 +296,7 @@ [(opened)]="rightLayoutOpened"> -
+
dashboard.mobile-layout - - {{ 'dashboard.autofill-height' | translate }} + + {{ 'dashboard.display-first-in-mobile-view' | translate }} - - dashboard.mobile-row-height - - - {{ 'dashboard.mobile-row-height-required' | translate }} - - - {{ 'dashboard.min-mobile-row-height-message' | translate }} - - - {{ 'dashboard.max-mobile-row-height-message' | translate }} - - +
+ + {{ 'dashboard.autofill-height' | translate }} + + + dashboard.mobile-row-height + + + {{ 'dashboard.mobile-row-height-required' | translate }} + + + {{ 'dashboard.min-mobile-row-height-message' | translate }} + + + {{ 'dashboard.max-mobile-row-height-message' | translate }} + + +
diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts index 4844211355..303964e099 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-settings-dialog.component.ts @@ -32,6 +32,7 @@ import { StatesControllerService } from './states/states-controller.service'; export interface DashboardSettingsDialogData { settings?: DashboardSettings; gridSettings?: GridSettings; + isRightLayout?: boolean; } @Component({ @@ -45,6 +46,7 @@ export class DashboardSettingsDialogComponent extends DialogComponent { if (mobileAutoFillHeightValue) { diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts index 1c04d05bae..b93a362f92 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.ts @@ -217,7 +217,8 @@ export class ManageDashboardLayoutsDialogComponent extends DialogComponent { if (data && data.gridSettings) { diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index ffe475d19d..8b92f9eb39 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -58,6 +58,7 @@ export interface GridSettings { autoFillHeight?: boolean; mobileAutoFillHeight?: boolean; mobileRowHeight?: number; + mobileDisplayLayoutFirst?: boolean; layoutDimension?: LayoutDimension; [key: string]: any; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ef660731bf..194eee430a 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1113,6 +1113,7 @@ "mobile-row-height-required": "Mobile row height value is required.", "min-mobile-row-height-message": "Only 5 pixels is allowed as minimum mobile row height value.", "max-mobile-row-height-message": "Only 200 pixels is allowed as maximum mobile row height value.", + "display-first-in-mobile-view": "Display first in mobile view", "title-settings": "Title settings", "display-title": "Display dashboard title", "title-color": "Title color", From c3e8c62ffcc51662f91e67f736e76f537a8270f2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 23 Jan 2024 12:36:20 +0100 Subject: [PATCH 004/410] RPC deleteOutdatedRpcByTenantId simple native query. the test added --- .../service/ttl/rpc/RpcCleanUpService.java | 2 +- .../thingsboard/server/dao/rpc/RpcDao.java | 3 +- .../server/dao/sql/rpc/JpaRpcDao.java | 4 +- .../server/dao/sql/rpc/RpcRepository.java | 6 +- .../server/dao/sql/rpc/JpaRpcDaoTest.java | 59 +++++++++++++++++++ 5 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java index c8683972d9..57ba35628c 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java @@ -68,7 +68,7 @@ public class RpcCleanUpService { long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays()); long expirationTime = System.currentTimeMillis() - ttl; - long totalRemoved = rpcDao.deleteOutdatedRpcByTenantId(tenantId, expirationTime); + int totalRemoved = rpcDao.deleteOutdatedRpcByTenantId(tenantId, expirationTime); if (totalRemoved > 0) { log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, new Date(expirationTime)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java index 79b73c69aa..d2e867e763 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java @@ -30,5 +30,6 @@ public interface RpcDao extends Dao { PageData findAllRpcByTenantId(TenantId tenantId, PageLink pageLink); - Long deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime); + int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java index 2c16db4d34..e1939a595c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -67,8 +68,9 @@ public class JpaRpcDao extends JpaAbstractDao implements RpcDao return DaoUtil.toPageData(rpcRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); } + @Transactional @Override - public Long deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { + public int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java index e04d000a65..fe7c95d249 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.rpc; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.rpc.RpcStatus; @@ -32,7 +33,8 @@ public interface RpcRepository extends JpaRepository { Page findAllByTenantId(UUID tenantId, Pageable pageable); - @Query(value = "WITH deleted AS (DELETE FROM rpc WHERE (tenant_id = :tenantId AND created_time < :expirationTime) IS TRUE RETURNING *) SELECT count(*) FROM deleted", + @Modifying + @Query(value = "DELETE FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime", nativeQuery = true) - Long deleteOutdatedRpcByTenantId(@Param("tenantId") UUID tenantId, @Param("expirationTime") Long expirationTime); + int deleteOutdatedRpcByTenantId(@Param("tenantId") UUID tenantId, @Param("expirationTime") Long expirationTime); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java new file mode 100644 index 0000000000..d2889dfb3c --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.rpc; + +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rpc.Rpc; +import org.thingsboard.server.common.data.rpc.RpcStatus; +import org.thingsboard.server.dao.AbstractJpaDaoTest; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JpaRpcDaoTest extends AbstractJpaDaoTest { + + @Autowired + JpaRpcDao rpcDao; + + @Test + public void deleteOutdated() { + Rpc rpc = new Rpc(); + rpc.setTenantId(TenantId.SYS_TENANT_ID); + rpc.setDeviceId(new DeviceId(UUID.randomUUID())); + rpc.setStatus(RpcStatus.QUEUED); + rpc.setRequest(JacksonUtil.toJsonNode("{}")); + rpcDao.saveAndFlush(rpc.getTenantId(), rpc); + + rpc.setId(null); + rpcDao.saveAndFlush(rpc.getTenantId(), rpc); + + TenantId tenantId = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); + rpc.setId(null); + rpc.setTenantId(tenantId); + rpc.setDeviceId(new DeviceId(UUID.randomUUID())); + rpcDao.saveAndFlush(rpc.getTenantId(), rpc); + + assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, 0L)).isEqualTo(0); + assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, Long.MAX_VALUE)).isEqualTo(2); + assertThat(rpcDao.deleteOutdatedRpcByTenantId(tenantId, System.currentTimeMillis() + 1)).isEqualTo(1); + } + +} From 053bcc0abe9b7798111548ea414ba850f2f8a78c Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 23 Jan 2024 12:36:40 +0100 Subject: [PATCH 005/410] Log Hibernate SQL queries in the dao test scope (commented) --- dao/src/test/resources/logback.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dao/src/test/resources/logback.xml b/dao/src/test/resources/logback.xml index 61397ec6f1..5e293b2982 100644 --- a/dao/src/test/resources/logback.xml +++ b/dao/src/test/resources/logback.xml @@ -10,6 +10,9 @@ + + + From 42238950817216d10aff688e3be2807b569e6323 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 7 Feb 2024 11:31:52 +0200 Subject: [PATCH 006/410] migrated assets used for storing queue statistics to queue_stats table --- .../main/data/upgrade/3.6.2/schema_update.sql | 20 +++++ .../server/controller/BaseController.java | 13 ++++ .../processor/asset/AssetEdgeProcessor.java | 3 +- .../entitiy/asset/DefaultTbAssetService.java | 14 ---- .../profile/DefaultTbAssetProfileService.java | 11 --- .../service/security/permission/Resource.java | 3 +- .../DefaultRuleEngineStatisticsService.java | 47 ++++++----- .../controller/AssetControllerTest.java | 3 - .../controller/BaseQueueControllerTest.java | 20 ++--- .../server/dao/queue/QueueStatsService.java | 35 +++++++++ .../server/common/data/EntityType.java | 3 +- .../server/common/data/id/QueueStatsId.java | 43 +++++++++++ .../server/common/data/queue/QueueStats.java | 37 +++++++++ .../server/dao/model/ModelConstants.java | 8 ++ .../dao/model/sql/QueueStatsEntity.java | 69 +++++++++++++++++ .../dao/queue/BaseQueueStatsService.java | 77 +++++++++++++++++++ .../server/dao/queue/QueueStatsDao.java | 30 ++++++++ .../service/validator/AssetDataValidator.java | 4 +- .../server/dao/sql/asset/AssetRepository.java | 2 +- .../server/dao/sql/asset/JpaAssetDao.java | 3 +- .../query/DefaultEntityQueryRepository.java | 2 + .../dao/sql/queue/JpaQueueStatsDao.java | 61 +++++++++++++++ .../dao/sql/queue/QueueStatsRepository.java | 30 ++++++++ .../main/resources/sql/schema-entities.sql | 9 +++ 24 files changed, 475 insertions(+), 72 deletions(-) create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/QueueStatsId.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/QueueStatsEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java diff --git a/application/src/main/data/upgrade/3.6.2/schema_update.sql b/application/src/main/data/upgrade/3.6.2/schema_update.sql index 6ae5e45134..1be58e4408 100644 --- a/application/src/main/data/upgrade/3.6.2/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.2/schema_update.sql @@ -28,3 +28,23 @@ ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS queue_name varchar(255); ALTER TABLE component_descriptor ADD COLUMN IF NOT EXISTS has_queue_name boolean DEFAULT false; -- RULE NODE QUEUE UPDATE END + +-- QUEUE STATS UPDATE START + +CREATE TABLE IF NOT EXISTS queue_stats ( + id uuid NOT NULL CONSTRAINT queue_stats_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + queue_name varchar(255) NOT NULL, + service_id varchar(255) NOT NULL, + CONSTRAINT queue_stats_name_unq_key UNIQUE (tenant_id, queue_name, service_id)); + +INSERT INTO queue_stats + SELECT id, created_time, tenant_id, split_part(name, '_', 1) AS queue_name, split_part(name, '_', 2) AS service_id + FROM asset + WHERE type = 'TbServiceQueue'; + +DELETE FROM asset WHERE type='TbServiceQueue'; +DELETE FROM asset_profile WHERE name ='TbServiceQueue'; + +-- QUEUE STATS UPDATE END \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index c40760dda3..5f406eed29 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -78,6 +78,7 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -96,6 +97,7 @@ import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -123,6 +125,7 @@ import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rpc.RpcService; @@ -307,6 +310,9 @@ public abstract class BaseController { @Autowired protected QueueService queueService; + @Autowired + protected QueueStatsService queueStatsService; + @Autowired protected EntitiesVersionControlService vcService; @@ -600,6 +606,9 @@ public abstract class BaseController { case QUEUE: checkQueueId(new QueueId(entityId.getId()), operation); return; + case QUEUE_STATS: + checkQueueStatsId(new QueueStatsId(entityId.getId()), operation); + return; default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } @@ -776,6 +785,10 @@ public abstract class BaseController { return queue; } + protected QueueStats checkQueueStatsId(QueueStatsId queueStatsId, Operation operation) throws ThingsboardException { + return checkEntityId(queueStatsId, queueStatsService::findQueueStatsById, operation); + } + protected I emptyId(EntityType entityType) { return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java index 203c6466e0..d8122e1b6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.dao.asset.BaseAssetService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -115,7 +114,7 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A case ASSIGNED_TO_CUSTOMER: case UNASSIGNED_FROM_CUSTOMER: Asset asset = assetService.findAssetById(edgeEvent.getTenantId(), assetId); - if (asset != null && !BaseAssetService.TB_SERVICE_QUEUE.equals(asset.getType())) { + if (asset != null) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); AssetUpdateMsg assetUpdateMsg = ((AssetMsgConstructor) assetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructAssetUpdatedMsg(msgType, asset); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index 58940318be..94d28b50d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -22,10 +22,8 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; @@ -34,30 +32,18 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; - -import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; @Service @AllArgsConstructor public class DefaultTbAssetService extends AbstractTbEntityService implements TbAssetService { private final AssetService assetService; - private final TbAssetProfileCache assetProfileCache; @Override public Asset save(Asset asset, User user) throws Exception { ActionType actionType = asset.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = asset.getTenantId(); try { - if (TB_SERVICE_QUEUE.equals(asset.getType())) { - throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } else if (asset.getAssetProfileId() != null) { - AssetProfile assetProfile = assetProfileCache.get(tenantId, asset.getAssetProfileId()); - if (assetProfile != null && TB_SERVICE_QUEUE.equals(assetProfile.getName())) { - throw new ThingsboardException("Unable to save asset with profile " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } - } Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); autoCommit(user, savedAsset.getId()); notificationEntityService.logEntityAction(tenantId, savedAsset.getId(), savedAsset, asset.getCustomerId(), diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/profile/DefaultTbAssetProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/profile/DefaultTbAssetProfileService.java index 832a9a3003..e5884b5eb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/profile/DefaultTbAssetProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/profile/DefaultTbAssetProfileService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.TenantId; @@ -31,8 +30,6 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; - @Service @TbCoreComponent @AllArgsConstructor @@ -46,14 +43,6 @@ public class DefaultTbAssetProfileService extends AbstractTbEntityService implem ActionType actionType = assetProfile.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = assetProfile.getTenantId(); try { - if (TB_SERVICE_QUEUE.equals(assetProfile.getName())) { - throw new ThingsboardException("Unable to save asset profile with name " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } else if (assetProfile.getId() != null) { - AssetProfile foundAssetProfile = assetProfileService.findAssetProfileById(tenantId, assetProfile.getId()); - if (foundAssetProfile != null && TB_SERVICE_QUEUE.equals(foundAssetProfile.getName())) { - throw new ThingsboardException("Updating asset profile with name " + TB_SERVICE_QUEUE + " is prohibited!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } - } AssetProfile savedAssetProfile = checkNotNull(assetProfileService.saveAssetProfile(assetProfile)); autoCommit(user, savedAssetProfile.getId()); tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedAssetProfile.getId(), diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 5aa869f2a8..770745f738 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -46,7 +46,8 @@ public enum Resource { QUEUE(EntityType.QUEUE), VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, - EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE); + EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), + QUEUE_STATS(EntityType.QUEUE_STATS); private final Set entityTypes; diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index ff69746bcc..fca9fce81a 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -21,15 +21,15 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -52,7 +52,6 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { - public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; public static final String RULE_ENGINE_EXCEPTION = "ruleEngineException"; public static final FutureCallback CALLBACK = new FutureCallback() { @Override @@ -68,10 +67,10 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS private final TbServiceInfoProvider serviceInfoProvider; private final TelemetrySubscriptionService tsService; - private final AssetService assetService; + private final QueueStatsService queueStatsService; private final ApiLimitService apiLimitService; private final Lock lock = new ReentrantLock(); - private final ConcurrentMap tenantQueueAssets = new ConcurrentHashMap<>(); + private final ConcurrentMap tenantQueueStats = new ConcurrentHashMap<>(); @Value("${queue.rule-engine.stats.max-error-message-length:4096}") private int maxErrorMessageLength; @@ -82,7 +81,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS ruleEngineStats.getTenantStats().forEach((id, stats) -> { try { TenantId tenantId = TenantId.fromUUID(id); - AssetId serviceAssetId = getServiceAssetId(tenantId, queueName); + QueueStatsId queueStatsId = getQueueStatsId(tenantId, queueName); if (stats.getTotalMsgCounter().get() > 0) { List tsList = stats.getCounters().entrySet().stream() .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) @@ -90,7 +89,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS if (!tsList.isEmpty()) { long ttl = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getQueueStatsTtlDays); ttl = TimeUnit.DAYS.toSeconds(ttl); - tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, ttl, CALLBACK); + tsService.saveAndNotifyInternal(tenantId, queueStatsId, tsList, ttl, CALLBACK); } } } catch (Exception e) { @@ -104,7 +103,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS TsKvEntry tsKv = new BasicTsKvEntry(e.getTs(), new JsonDataEntry(RULE_ENGINE_EXCEPTION, e.toJsonString(maxErrorMessageLength))); long ttl = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getRuleEngineExceptionsTtlDays); ttl = TimeUnit.DAYS.toSeconds(ttl); - tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), ttl, CALLBACK); + tsService.saveAndNotifyInternal(tenantId, getQueueStatsId(tenantId, queueName), Collections.singletonList(tsKv), ttl, CALLBACK); } catch (Exception e2) { if (!"Asset is referencing to non-existent tenant!".equalsIgnoreCase(e2.getMessage())) { log.debug("[{}] Failed to store the statistics", tenantId, e2); @@ -113,30 +112,30 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS }); } - private AssetId getServiceAssetId(TenantId tenantId, String queueName) { + private QueueStatsId getQueueStatsId(TenantId tenantId, String queueName) { TenantQueueKey key = new TenantQueueKey(tenantId, queueName); - AssetId assetId = tenantQueueAssets.get(key); - if (assetId == null) { + QueueStatsId queueStatsId = tenantQueueStats.get(key); + if (queueStatsId == null) { lock.lock(); try { - assetId = tenantQueueAssets.get(key); - if (assetId == null) { - Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId()); - if (asset == null) { - asset = new Asset(); - asset.setTenantId(tenantId); - asset.setName(queueName + "_" + serviceInfoProvider.getServiceId()); - asset.setType(TB_SERVICE_QUEUE); - asset = assetService.saveAsset(asset); + queueStatsId = tenantQueueStats.get(key); + if (queueStatsId == null) { + QueueStats queueStats = queueStatsService.findByTenantIdAndNameAndServiceId(tenantId, queueName , serviceInfoProvider.getServiceId()); + if (queueStats == null) { + queueStats = new QueueStats(); + queueStats.setTenantId(tenantId); + queueStats.setQueueName(queueName); + queueStats.setServiceId(serviceInfoProvider.getServiceId()); + queueStats = queueStatsService.save(tenantId, queueStats); } - assetId = asset.getId(); - tenantQueueAssets.put(key, assetId); + queueStatsId = queueStats.getId(); + tenantQueueStats.put(key, queueStatsId); } } finally { lock.unlock(); } } - return assetId; + return queueStatsId; } @Data diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java index b41c8c3e78..a3e1e6cccf 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java @@ -50,7 +50,6 @@ import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import java.util.ArrayList; import java.util.List; @@ -567,8 +566,6 @@ public class AssetControllerTest extends AbstractControllerTest { savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, cntEntity, cntEntity, cntEntity); - loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); - assets.sort(idComparator); loadedAssets.sort(idComparator); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 772a3c5ecb..87688a797f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -25,7 +25,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -34,12 +33,13 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.ProcessingStrategy; import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.gen.transport.TransportProtos; @@ -50,6 +50,7 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingRes import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; import org.thingsboard.server.service.stats.RuleEngineStatisticsService; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -66,7 +67,6 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; @DaoSqlTest @TestPropertySource(properties = { @@ -81,7 +81,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { @SpyBean private TimeseriesDao timeseriesDao; @Autowired - private AssetService assetService; + private QueueStatsService queueStatsService; @Test public void testQueueWithServiceTypeRE() throws Exception { @@ -176,16 +176,16 @@ public class BaseQueueControllerTest extends AbstractControllerTest { }); ruleEngineStatisticsService.reportQueueStats(System.currentTimeMillis(), testStats); - Asset serviceAsset = assetService.findAssetsByTenantIdAndType(tenantId, TB_SERVICE_QUEUE, new PageLink(100)).getData() - .stream().filter(asset -> asset.getName().startsWith(queue.getName())) - .findFirst().get(); + List queueStatsList = queueStatsService.findByTenantId(tenantId); + assertThat(queueStatsList).hasSize(1); + QueueStats queueStats = queueStatsList.get(0); ArgumentCaptor ttlCaptor = ArgumentCaptor.forClass(Long.class); - verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> { + verify(timeseriesDao).save(eq(tenantId), eq(queueStats.getId()), argThat(tsKvEntry -> { return tsKvEntry.getKey().equals(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS) && tsKvEntry.getLongValue().get().equals(5L); }), ttlCaptor.capture()); - verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> { + verify(timeseriesDao).save(eq(tenantId), eq(queueStats.getId()), argThat(tsKvEntry -> { return tsKvEntry.getKey().equals(TbRuleEngineConsumerStats.FAILED_MSGS) && tsKvEntry.getLongValue().get().equals(5L); }), ttlCaptor.capture()); @@ -193,7 +193,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { assertThat(usedTtl).isEqualTo(TimeUnit.DAYS.toSeconds(queueStatsTtlDays)); }); - verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> { + verify(timeseriesDao).save(eq(tenantId), eq(queueStats.getId()), argThat(tsKvEntry -> { return tsKvEntry.getKey().equals(DefaultRuleEngineStatisticsService.RULE_ENGINE_EXCEPTION) && tsKvEntry.getJsonValue().get().equals(ruleEngineException.toJsonString(0)); }), ttlCaptor.capture()); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java new file mode 100644 index 0000000000..e7166c4fd6 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.queue; + +import org.thingsboard.server.common.data.id.QueueStatsId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.dao.entity.EntityDaoService; + +import java.util.List; + +public interface QueueStatsService extends EntityDaoService { + + QueueStats save(TenantId tenantId, QueueStats queueStats); + + QueueStats findQueueStatsById(TenantId tenantId, QueueStatsId queueStatsId); + + QueueStats findByTenantIdAndNameAndServiceId(TenantId tenantId, String queueStatsName, String serviceId); + + List findByTenantId(TenantId tenantId); + +} \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index fb4fe1011e..bb43bf4d11 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -58,7 +58,8 @@ public enum EntityType { NOTIFICATION_TEMPLATE (30), NOTIFICATION_REQUEST (31), NOTIFICATION (32), - NOTIFICATION_RULE (33); + NOTIFICATION_RULE (33), + QUEUE_STATS(34); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/QueueStatsId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/QueueStatsId.java new file mode 100644 index 0000000000..04629007a9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/QueueStatsId.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModelProperty; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public class QueueStatsId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public QueueStatsId(@JsonProperty("id") UUID id) { + super(id); + } + + public static QueueStatsId fromString(String queueId) { + return new QueueStatsId(UUID.fromString(queueId)); + } + + @ApiModelProperty(position = 2, required = true, value = "string", example = "QUEUE_STATS", allowableValues = "QUEUE_STATS") + @Override + public EntityType getEntityType() { + return EntityType.QUEUE_STATS; + } +} \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java new file mode 100644 index 0000000000..b6224d4167 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.queue; + +import lombok.Data; +import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.QueueStatsId; + +@Data +public class QueueStats extends BaseDataWithAdditionalInfo implements HasTenantId { + private TenantId tenantId; + private String queueName; + private String serviceId; + + public QueueStats() { + } + + public QueueStats(QueueStatsId id) { + super(id); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 2fe56f5d1f..166bdbc0ee 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -610,6 +610,14 @@ public class ModelConstants { public static final String QUEUE_TABLE_NAME = "queue"; public static final String QUEUE_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY; + /** + * Tenant queue stats constants. + */ + public static final String QUEUE_STATS_TABLE_NAME = "queue_stats"; + public static final String QUEUE_STATS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; + public static final String QUEUE_STATS_QUEUE_NAME_PROPERTY = "queue_name"; + public static final String QUEUE_STATS_SERVICE_ID_PROPERTY = "service_id"; + /** * Notification constants */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/QueueStatsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/QueueStatsEntity.java new file mode 100644 index 0000000000..1cdbd2bc5e --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/QueueStatsEntity.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.QueueStatsId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = ModelConstants.QUEUE_STATS_TABLE_NAME) +public class QueueStatsEntity extends BaseSqlEntity { + + @Column(name = ModelConstants.QUEUE_STATS_TENANT_ID_PROPERTY) + private UUID tenantId; + + @Column(name = ModelConstants.QUEUE_STATS_QUEUE_NAME_PROPERTY) + private String queueName; + + @Column(name = ModelConstants.QUEUE_STATS_SERVICE_ID_PROPERTY) + private String serviceId; + + public QueueStatsEntity() { + } + + public QueueStatsEntity(QueueStats queueStats) { + if (queueStats.getId() != null) { + this.setId(queueStats.getId().getId()); + } + this.setCreatedTime(queueStats.getCreatedTime()); + this.tenantId = DaoUtil.getId(queueStats.getTenantId()); + this.queueName = queueStats.getQueueName(); + this.serviceId = queueStats.getServiceId(); + } + + @Override + public QueueStats toData() { + QueueStats queueStats = new QueueStats(new QueueStatsId(getUuid())); + queueStats.setCreatedTime(createdTime); + queueStats.setTenantId(new TenantId(tenantId)); + queueStats.setQueueName(queueName); + queueStats.setServiceId(serviceId); + return queueStats; + } +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java new file mode 100644 index 0000000000..b58d4a162f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.queue; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.QueueStatsId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.dao.entity.AbstractEntityService; + +import java.util.List; +import java.util.Optional; + +import static org.thingsboard.server.dao.service.Validator.validateId; + +@Service("QueueStatsDaoService") +@Slf4j +@RequiredArgsConstructor +public class BaseQueueStatsService extends AbstractEntityService implements QueueStatsService { + + @Autowired + private QueueStatsDao queueStatsDao; + + @Override + public QueueStats save(TenantId tenantId, QueueStats queueStats) { + return queueStatsDao.save(tenantId, queueStats); + } + + @Override + public QueueStats findQueueStatsById(TenantId tenantId, QueueStatsId queueStatsId) { + log.trace("Executing findQueueStatsById [{}]", queueStatsId); + validateId(queueStatsId, "Incorrect queueStatsId " + queueStatsId); + return queueStatsDao.findById(tenantId, queueStatsId.getId()); + } + + @Override + public QueueStats findByTenantIdAndNameAndServiceId(TenantId tenantId, String queueName, String serviceId) { + log.trace("Executing findByTenantIdAndNameAndServiceId, tenantId: [{}], queueName: [{}], serviceId: [{}]", tenantId, queueName, serviceId); + return queueStatsDao.findByTenantIdQueueNameAndServiceId(tenantId, queueName, serviceId); + } + + @Override + public List findByTenantId(TenantId tenantId) { + log.trace("Executing findByTenantId, tenantId: [{}]", tenantId); + return queueStatsDao.findByTenantId(tenantId); + } + + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findQueueStatsById(tenantId, new QueueStatsId(entityId.getId()))); + } + + @Override + public EntityType getEntityType() { + return null; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java new file mode 100644 index 0000000000..fc83891036 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.queue; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.dao.Dao; + +import java.util.List; + +public interface QueueStatsDao extends Dao { + + QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String name, String serviceId); + + List findByTenantId(TenantId tenantId); + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java index 5ef0b7a2ba..2d0cdbd248 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java @@ -47,9 +47,7 @@ public class AssetDataValidator extends DataValidator { @Override protected void validateCreate(TenantId tenantId, Asset asset) { - if (!BaseAssetService.TB_SERVICE_QUEUE.equals(asset.getType())) { - validateNumberOfEntitiesPerTenant(tenantId, EntityType.ASSET); - } + validateNumberOfEntitiesPerTenant(tenantId, EntityType.ASSET); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index bbe212fb0b..f65e0db1c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -189,7 +189,7 @@ public interface AssetRepository extends JpaRepository, Expor @Param("searchText") String searchText, Pageable pageable); - Long countByTenantIdAndTypeIsNot(UUID tenantId, String type); + Long countByTenantId(UUID tenantId); @Query("SELECT externalId FROM AssetEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index b77559be7b..0ef4370f30 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -43,7 +43,6 @@ import java.util.Optional; import java.util.UUID; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; -import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; /** * Created by Valerii Sosliuk on 5/19/2017. @@ -244,7 +243,7 @@ public class JpaAssetDao extends JpaAbstractDao implements A @Override public Long countByTenantId(TenantId tenantId) { - return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE); + return assetRepository.countByTenantId(tenantId.getId()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index d2b2567bd7..896fde6644 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -244,6 +244,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityTableMap.put(EntityType.DEVICE_PROFILE, "device_profile"); entityTableMap.put(EntityType.ASSET_PROFILE, "asset_profile"); entityTableMap.put(EntityType.TENANT_PROFILE, "tenant_profile"); + entityTableMap.put(EntityType.QUEUE_STATS, "queue_stats"); entityNameColumns.put(EntityType.DEVICE, "name"); entityNameColumns.put(EntityType.CUSTOMER, "title"); @@ -262,6 +263,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityNameColumns.put(EntityType.TB_RESOURCE, "search_text"); entityNameColumns.put(EntityType.EDGE, "name"); entityNameColumns.put(EntityType.QUEUE, "name"); + entityNameColumns.put(EntityType.QUEUE_STATS, "queue_name"); } public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java new file mode 100644 index 0000000000..29853ce611 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.queue; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sql.QueueStatsEntity; +import org.thingsboard.server.dao.queue.QueueStatsDao; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Component +@SqlDao +public class JpaQueueStatsDao extends JpaAbstractDao implements QueueStatsDao { + + @Autowired + private QueueStatsRepository queueStatsRepository; + + @Override + protected Class getEntityClass() { + return QueueStatsEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return queueStatsRepository; + } + + @Override + public QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String name, String serviceId) { + return DaoUtil.getData(queueStatsRepository.findByTenantIdAndQueueNameAndServiceId(tenantId.getId(), name, serviceId)); + } + + @Override + public List findByTenantId(TenantId tenantId) { + return DaoUtil.convertDataList(queueStatsRepository.findByTenantId(tenantId.getId())); + } + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java new file mode 100644 index 0000000000..1fffd0913a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.queue; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.QueueStatsEntity; + +import java.util.List; +import java.util.UUID; + +public interface QueueStatsRepository extends JpaRepository { + + QueueStatsEntity findByTenantIdAndQueueNameAndServiceId(UUID tenantId, String name, String serviceId); + + List findByTenantId(UUID tenantId); + +} \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 2352ea2eb3..dab74ec1ab 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -884,3 +884,12 @@ CREATE TABLE IF NOT EXISTS alarm_types ( CONSTRAINT tenant_id_type_unq_key UNIQUE (tenant_id, type), CONSTRAINT fk_entity_tenant_id FOREIGN KEY (tenant_id) REFERENCES tenant(id) ON DELETE CASCADE ); + +CREATE TABLE IF NOT EXISTS queue_stats ( + id uuid NOT NULL CONSTRAINT queue_stats_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + queue_name varchar(255) NOT NULL, + service_id varchar(255) NOT NULL, + CONSTRAINT queue_stats_name_unq_key UNIQUE (tenant_id, queue_name, service_id) +); \ No newline at end of file From e5311984bba0d6449e6db65c9fdfac22cf2e98b7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 Feb 2024 15:24:40 +0200 Subject: [PATCH 007/410] Housekeeper API --- .../DefaultHousekeeperService.java | 144 ++++++++++++++++++ .../HousekeeperReprocessingService.java | 134 ++++++++++++++++ .../AttributesDeletionTaskProcessor.java | 41 +++++ .../EntityDeletionTaskProcessor.java | 37 +++++ .../EventsDeletionTaskProcessor.java | 38 +++++ .../processor/HousekeeperTaskProcessor.java | 27 ++++ .../TelemetryDeletionTaskProcessor.java | 41 +++++ .../src/main/resources/thingsboard.yml | 7 + common/proto/src/main/proto/queue.proto | 8 + .../queue/common/AbstractTbQueueTemplate.java | 4 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../provider/AwsSqsMonolithQueueFactory.java | 21 +++ .../provider/AwsSqsTbCoreQueueFactory.java | 21 +++ .../InMemoryMonolithQueueFactory.java | 20 +++ .../provider/KafkaMonolithQueueFactory.java | 51 ++++++- .../provider/KafkaTbCoreQueueFactory.java | 50 ++++++ .../provider/PubSubMonolithQueueFactory.java | 21 +++ .../provider/PubSubTbCoreQueueFactory.java | 21 +++ .../RabbitMqMonolithQueueFactory.java | 21 +++ .../provider/RabbitMqTbCoreQueueFactory.java | 20 +++ .../ServiceBusMonolithQueueFactory.java | 21 +++ .../ServiceBusTbCoreQueueFactory.java | 21 +++ .../queue/provider/TbCoreQueueFactory.java | 11 ++ .../provider/TbCoreQueueProducerProvider.java | 17 +++ .../provider/TbQueueProducerProvider.java | 6 + .../TbRuleEngineProducerProvider.java | 11 ++ .../TbTransportQueueProducerProvider.java | 10 ++ .../TbVersionControlProducerProvider.java | 10 ++ .../queue/settings/TbQueueCoreSettings.java | 6 + .../server/dao/device/DeviceServiceImpl.java | 5 + .../dao/entity/AbstractEntityService.java | 6 + .../dao/housekeeper/HousekeeperService.java | 24 +++ .../dao/housekeeper/data/HousekeeperTask.java | 51 +++++++ .../housekeeper/data/HousekeeperTaskType.java | 23 +++ 34 files changed, 951 insertions(+), 3 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java new file mode 100644 index 0000000000..7c7e72b511 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.housekeeper.HousekeeperService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultHousekeeperService implements HousekeeperService { + + private final Map taskProcessors; + + private final TbQueueConsumer> consumer; + private final TbQueueProducer> producer; + private final HousekeeperReprocessingService reprocessingService; + private final DataDecodingEncodingService dataDecodingEncodingService; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private boolean stopped; + + public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider, + DataDecodingEncodingService dataDecodingEncodingService, List taskProcessors) { + this.consumer = queueFactory.createHousekeeperMsgConsumer(); + this.producer = producerProvider.getHousekeeperMsgProducer(); + this.reprocessingService = reprocessingService; + this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); + this.dataDecodingEncodingService = dataDecodingEncodingService; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + continue; + } + + for (TbProtoQueueMsg msg : msgs) { + try { + processTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper service"); + } + + protected void processTask(TbProtoQueueMsg msg) { + HousekeeperTask task = dataDecodingEncodingService.decode(msg.getValue().getTask().getValue().toByteArray()).get(); + HousekeeperTaskProcessor taskProcessor = taskProcessors.get(task.getTaskType()); + if (taskProcessor == null) { + log.error("Unsupported task type {}: {}", task.getTaskType(), task); + return; + } + + log.info("[{}] Processing task: {}", task.getTenantId(), task); + try { + taskProcessor.process(task); + } catch (Exception e) { + log.error("[{}] Task processing failed: {}", task.getTenantId(), task, e); + reprocessingService.submitForReprocessing(msg); + } + } + + @Override + public void submitTask(HousekeeperTask task) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), ToHousekeeperServiceMsg.newBuilder() + .setTask(HousekeeperTaskProto.newBuilder() + .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .build()) + .build()), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper service"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java new file mode 100644 index 0000000000..234cd07c36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToLong; +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.longToBytes; + +@TbCoreComponent +@Service +@Slf4j +public class HousekeeperReprocessingService { + + private final DefaultHousekeeperService housekeeperService; + private final TbQueueProducerProvider producerProvider; + private final TbQueueConsumer> consumer; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private final long startTs = System.currentTimeMillis(); + private boolean stopped; + // todo: stats + + public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider) { + this.housekeeperService = housekeeperService; + this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer(); + this.producerProvider = producerProvider; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + stop(); + return; + } + + for (TbProtoQueueMsg msg : msgs) { + long msgTs = Uuids.unixTimestamp(msg.getKey()); + if (msgTs >= startTs) { + stop(); + return; // fixme: we should commit already reprocessed messages + } + + try { + reprocessTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper tasks reprocessing"); + } + + private void reprocessTask(TbProtoQueueMsg msg) { + housekeeperService.processTask(msg);// fixme: or should we submit to queue? + } + + public void submitForReprocessing(TbProtoQueueMsg msg) { + TbQueueMsgHeaders msgHeaders = msg.getHeaders(); + long reprocessingAttempts = Optional.ofNullable(msgHeaders.get("reprocessingAttempts")) + .map(header -> bytesToLong(header)) + .orElse(0L); + reprocessingAttempts++; + msgHeaders.put("reprocessingAttempts", longToBytes(reprocessingAttempts)); + + var producer = producerProvider.getHousekeeperDelayedMsgProducer(); + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), msg.getValue(), msgHeaders), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper tasks reprocessing"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java new file mode 100644 index 0000000000..34bd95a901 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final AttributesService attributesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// attributesService.removeAll(task.getTenantId(), task.getEntityId(), DataConstants.CLIENT_SCOPE); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java new file mode 100644 index 0000000000..e1b0877d2c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { + + @Override + public void process(HousekeeperTask task) throws Exception { + + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java new file mode 100644 index 0000000000..7b8fdb1242 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { + private final EventService eventService; + + @Override + public void process(HousekeeperTask task) throws Exception { + eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_EVENTS; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java new file mode 100644 index 0000000000..bfa7466dc3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +public interface HousekeeperTaskProcessor { + + void process(HousekeeperTask task) throws Exception; + + HousekeeperTaskType getTaskType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java new file mode 100644 index 0000000000..feb5b0ddc1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +@Component +@RequiredArgsConstructor +public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// timeseriesService.remove() +// timeseriesService.removeAllLatest(); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 70a5146bb7..17b4c85cfb 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1383,6 +1383,9 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_housekeeper: + - key: max.poll.records + value: "1" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1404,6 +1407,7 @@ queue: ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1559,6 +1563,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + topic: "tb_housekeeper" + poll-interval-ms: "1000" vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4da2883291..0dbd96a53d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1391,3 +1391,11 @@ message LifecycleEventProto { bool success = 7; string error = 8; } + +message ToHousekeeperServiceMsg { + HousekeeperTaskProto task = 1; +} + +message HousekeeperTaskProto { + bytes value = 1; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index f11eaaef48..9340cafa67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -46,13 +46,13 @@ public class AbstractTbQueueTemplate { return new String(data, StandardCharsets.UTF_8); } - protected static byte[] longToBytes(long x) { + public static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - protected static long bytesToLong(byte[] bytes) { + public static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index d23d92ee26..d3a5eb821d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -43,6 +43,8 @@ public class TbKafkaTopicConfigs { private String fwUpdatesProperties; @Value("${queue.kafka.topic-properties.version-control:}") private String vcProperties; + @Value("${queue.kafka.topic-properties.housekeeper:}") + private String housekeeperProperties; @Getter private Map coreConfigs; @@ -62,6 +64,8 @@ public class TbKafkaTopicConfigs { private Map fwUpdatesConfigs; @Getter private Map vcConfigs; + @Getter + private Map housekeeperConfigs; @PostConstruct private void init() { @@ -76,6 +80,7 @@ public class TbKafkaTopicConfigs { jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1"); fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); vcConfigs = PropertyUtils.getProps(vcProperties); + housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 8a127384be..5a61bc1d80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -226,6 +227,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 6c69b2ec10..a1be25f29d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -205,6 +206,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index a681038aa9..1ee8b404a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -160,6 +160,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 662d29af5b..5ee9b6f53b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -39,8 +40,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -82,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -115,6 +117,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -346,6 +349,52 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("monolith-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index db6ee52181..c01ba32f70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -80,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -112,6 +114,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -303,6 +306,53 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("tb-core-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 62fe40f853..a63a9fc16b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -227,6 +228,26 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic()); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 0e7e1ba241..5f1f282e39 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1b7f705175..802b08773d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -224,6 +225,26 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e0d0ea3b3a..ceb6e03ca5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -198,6 +198,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 359cbf67b8..1e9c1cadaf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -223,6 +224,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index d2ef072ffa..e39df45b82 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 5aa7866033..279a469415 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -130,4 +132,13 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { * @return */ TbQueueProducer> createVersionControlMsgProducer(); + + TbQueueProducer> createHousekeeperMsgProducer(); + + TbQueueConsumer> createHousekeeperMsgConsumer(); + + TbQueueProducer> createHousekeeperDelayedMsgProducer(); + + TbQueueConsumer> createHousekeeperDelayedMsgConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 5a99f361b4..4fa22cb7e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -41,6 +43,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; + private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toHousekeeperDelayed; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -55,6 +59,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); + this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer(); } @Override @@ -91,4 +97,15 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { return toVersionControl; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + return toHousekeeperDelayed; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 4dbdca4375..5cdf390f47 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -78,4 +79,9 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbVersionControlMsgProducer(); + + TbQueueProducer> getHousekeeperMsgProducer(); + + TbQueueProducer> getHousekeeperDelayedMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index e4d98796f4..c5ddf57fc0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -89,4 +89,15 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return null; // fixme + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index c83b905a7c..e8dd7431a6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -85,4 +85,14 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index dbbaa9f072..7ddd2109ba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -81,4 +81,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 0a8c12eb8d..4edb688235 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -34,6 +34,12 @@ public class TbQueueCoreSettings { @Value("${queue.core.usage-stats-topic:tb_usage_stats}") private String usageStatsTopic; + @Value("${queue.core.housekeeper.topic:tb_housekeeper}") + private String housekeeperTopic; + + @Value("${queue.core.housekeeper.topic:tb_housekeeper.delayed}") + private String housekeeperDelayedTopic; + @Value("${queue.core.partitions}") private int partitions; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f190923167..f0e5dc1a4a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -75,6 +75,7 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -328,6 +329,10 @@ public class DeviceServiceImpl extends AbstractCachedEntityService Date: Wed, 7 Feb 2024 16:12:11 +0200 Subject: [PATCH 008/410] Submit cleanup tasks on DeleteEntityEvent --- .../EntityAlarmsDeletionTaskProcessor.java | 40 +++++++++++++++++++ .../server/dao/alarm/AlarmService.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 1 - .../server/dao/asset/BaseAssetService.java | 3 -- .../dao/customer/CustomerServiceImpl.java | 1 - .../dao/dashboard/DashboardServiceImpl.java | 1 - .../dao/device/DeviceProfileServiceImpl.java | 1 - .../server/dao/device/DeviceServiceImpl.java | 8 ---- .../server/dao/edge/EdgeServiceImpl.java | 7 +--- .../dao/entity/AbstractEntityService.java | 31 +++++++++++--- .../dao/entityview/EntityViewServiceImpl.java | 1 - .../dao/housekeeper/data/HousekeeperTask.java | 9 +++++ .../housekeeper/data/HousekeeperTaskType.java | 4 +- .../server/dao/rule/BaseRuleChainService.java | 7 ++-- .../dao/tenant/TenantProfileServiceImpl.java | 1 - .../server/dao/tenant/TenantServiceImpl.java | 1 - .../server/dao/user/UserServiceImpl.java | 1 - 18 files changed, 84 insertions(+), 38 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java new file mode 100644 index 0000000000..f9e61fc1cb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final AlarmService alarmService; + + @Override + public void process(HousekeeperTask task) throws Exception { + alarmService.deleteEntityAlarmRecords(task.getTenantId(), task.getEntityId()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITY_ALARMS; + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 203fe8e744..7a55348bed 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -106,7 +106,7 @@ public interface AlarmService extends EntityDaoService { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); - void deleteEntityAlarmRelations(TenantId tenantId, EntityId entityId); + void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); void deleteEntityAlarmRecordsByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 72e1b9c7bf..a14c172a57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -189,7 +189,6 @@ public class BaseAlarmService extends AbstractCachedEntityService event) { + TenantId tenantId = event.getTenantId(); + EntityId entityId = event.getEntityId(); + log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); + + cleanUpRelatedData(tenantId, entityId); + if (EntityType.USER.equals(entityId.getEntityType())) { +// housekeeperService.submitTask(HousekeeperTask.unassignAlarms(tenantId, entityId)); +// unassignDeletedUserAlarms(tenantId, (User) event.getEntity(), event.getTs()); + } + } + + protected void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { + // todo: skipped entities list + relationService.deleteEntityRelations(tenantId, entityId); + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + } + protected void createRelation(TenantId tenantId, EntityRelation relation) { log.debug("Creating relation: {}", relation); relationService.saveRelation(tenantId, relation); @@ -79,11 +103,6 @@ public abstract class AbstractEntityService { relationService.deleteRelation(tenantId, relation); } - protected void deleteEntityRelations(TenantId tenantId, EntityId entityId) { - relationService.deleteEntityRelations(tenantId, entityId); - alarmService.deleteEntityAlarmRelations(tenantId, entityId); - } - protected static Optional extractConstraintViolationException(Exception t) { if (t instanceof ConstraintViolationException) { return Optional.of((ConstraintViolationException) t); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index ed962ce84c..6940b42794 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -331,7 +331,6 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { - deleteEntityRelations(tenantId, existingNode.getId()); + cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { @@ -771,7 +771,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); for (var node : ruleNodes) { - deleteEntityRelations(tenantId, node.getId()); + cleanUpRelatedData(tenantId, node.getId()); } ruleNodeDao.deleteByIdIn(ruleNodeIds); } @@ -783,7 +783,6 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); } - deleteEntityRelations(tenantId, ruleChainId); } @@ -820,8 +819,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } private void deleteRuleNode(TenantId tenantId, EntityId entityId) { - deleteEntityRelations(tenantId, entityId); ruleNodeDao.removeById(tenantId, entityId.getId()); + cleanUpRelatedData(tenantId, entityId); } private final PaginatedRemover tenantRuleChainsRemover = diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 31a419a5d0..2462ca19d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -130,7 +130,6 @@ public class TenantProfileServiceImpl extends AbstractCachedEntityService Date: Wed, 7 Feb 2024 16:50:03 +0200 Subject: [PATCH 009/410] Alarms unassign housekeeper task; remove in-memory housekeeper service --- .../entitiy/alarm/DefaultTbAlarmService.java | 16 +-- .../service/entitiy/alarm/TbAlarmService.java | 2 +- .../InMemoryHouseKeeperServiceService.java | 103 ------------------ .../AlarmsUnassignTaskProcessor.java | 41 +++++++ .../AttributesDeletionTaskProcessor.java | 3 +- .../EntityAlarmsDeletionTaskProcessor.java | 2 +- .../EntityDeletionTaskProcessor.java | 2 +- .../EventsDeletionTaskProcessor.java | 2 +- .../processor/HousekeeperTaskProcessor.java | 4 +- .../TelemetryDeletionTaskProcessor.java | 2 +- .../alarm/DefaultTbAlarmServiceTest.java | 2 +- .../dao/entity/AbstractEntityService.java | 6 +- .../data/AlarmsUnassignHousekeeperTask.java | 18 +-- .../dao/housekeeper/data/HousekeeperTask.java | 19 ++-- 14 files changed, 82 insertions(+), 140 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java rename common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java => dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java (59%) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 5305018b84..deefdaa614 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -176,16 +176,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { + public List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) { List totalAlarmIds = new ArrayList<>(); PageLink pageLink = new PageLink(100, 0, null, new SortOrder("id", SortOrder.Direction.ASC)); while (true) { - PageData pageData = alarmService.findAlarmIdsByAssigneeId(user.getTenantId(), user.getId(), pageLink); + PageData pageData = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, pageLink); List alarmIds = pageData.getData(); if (alarmIds.isEmpty()) { break; } - processAlarmsUnassignment(tenantId, user, alarmIds, unassignTs); + processAlarmsUnassignment(tenantId, userId, userTitle, alarmIds, unassignTs); totalAlarmIds.addAll(alarmIds); pageLink = pageLink.nextPageLink(); } @@ -204,16 +204,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb return ts > 0 ? ts : System.currentTimeMillis(); } - private void processAlarmsUnassignment(TenantId tenantId, User user, List alarmIds, long unassignTs) { + private void processAlarmsUnassignment(TenantId tenantId, UserId userId, String userTitle, List alarmIds, long unassignTs) { for (AlarmId alarmId : alarmIds) { - log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, user.getId()); - AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(user.getTenantId(), alarmId, unassignTs); + log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, userId); + AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(tenantId, alarmId, unassignTs); if (!result.isSuccessful()) { - log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, user.getId()); + log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, userId); continue; } if (result.isModified()) { - String comment = String.format("Alarm was unassigned because user %s - was deleted", user.getTitle()); + String comment = String.format("Alarm was unassigned because user %s - was deleted", userTitle); addSystemAlarmComment(result.getAlarm(), null, "ASSIGN", comment); notificationEntityService.logEntityAction(result.getAlarm().getTenantId(), result.getAlarm().getOriginator(), result.getAlarm(), result.getAlarm().getCustomerId(), ActionType.ALARM_UNASSIGNED, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index 34dc3d11f2..c975291b58 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -41,7 +41,7 @@ public interface TbAlarmService { AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws ThingsboardException; - List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs); Boolean delete(Alarm alarm, User user); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java deleted file mode 100644 index 4870990606..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.housekeeper; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.transaction.event.TransactionalEventListener; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; -import org.thingsboard.server.dao.housekeeper.HouseKeeperService; -import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@RequiredArgsConstructor -@Slf4j -public class InMemoryHouseKeeperServiceService implements HouseKeeperService { - - final TbAlarmService alarmService; - - ListeningExecutorService executor; - - AtomicInteger queueSize = new AtomicInteger(); - AtomicInteger totalProcessedCounter = new AtomicInteger(); - - @PostConstruct - public void init() { - log.debug("Starting HouseKeeper service"); - executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper"))); - } - - @PreDestroy - public void destroy() { - if (executor != null) { - log.debug("Stopping HouseKeeper service"); - executor.shutdown(); - } - } - - @TransactionalEventListener(fallbackExecution = true) - public void handleEvent(DeleteEntityEvent event) { - log.trace("[{}] DeleteEntityEvent handler: {}", event.getTenantId(), event); - EntityId entityId = event.getEntityId(); - if (EntityType.USER.equals(entityId.getEntityType())) { - unassignDeletedUserAlarms(event.getTenantId(), (User) event.getEntity(), event.getTs()); - } - } - - @Override - public ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { - log.debug("[{}][{}] unassignDeletedUserAlarms submitting, pending queue size: {} ", tenantId, user.getId().getId(), queueSize.get()); - queueSize.incrementAndGet(); - ListenableFuture> future = executor.submit(() -> alarmService.unassignDeletedUserAlarms(tenantId, user, unassignTs)); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(List alarmIds) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.debug("[{}][{}] unassignDeletedUserAlarms finished, pending queue size: {}, total processed count: {} ", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get()); - } - - @Override - public void onFailure(Throwable throwable) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.error("[{}][{}] unassignDeletedUserAlarms failed, pending queue size: {}, total processed count: {}", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get(), throwable); - } - }, MoreExecutors.directExecutor()); - return future; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java new file mode 100644 index 0000000000..54bf7989af --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.dao.housekeeper.data.AlarmsUnassignHousekeeperTask; +import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; + +@Component +@RequiredArgsConstructor +public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor { + + private final TbAlarmService alarmService; + + @Override + public void process(AlarmsUnassignHousekeeperTask task) throws Exception { + alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.UNASSIGN_ALARMS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index 34bd95a901..169ac3e573 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -17,14 +17,13 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AttributesService attributesService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java index f9e61fc1cb..a91f2262f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -23,7 +23,7 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AlarmService alarmService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java index e1b0877d2c..deb693c466 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java @@ -22,7 +22,7 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { @Override public void process(HousekeeperTask task) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 7b8fdb1242..56edf01637 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -23,7 +23,7 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { private final EventService eventService; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java index bfa7466dc3..7daa99540e 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -18,9 +18,9 @@ package org.thingsboard.server.service.housekeeper.processor; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; -public interface HousekeeperTaskProcessor { +public interface HousekeeperTaskProcessor { - void process(HousekeeperTask task) throws Exception; + void process(T task) throws Exception; HousekeeperTaskType getTaskType(); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index feb5b0ddc1..2a491148ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -23,7 +23,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; @Component @RequiredArgsConstructor -public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { private final TimeseriesService timeseriesService; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 5d493193db..38b64d50ed 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -173,7 +173,7 @@ public class DefaultTbAlarmServiceTest { User user = new User(); user.setEmail("testEmail@gmail.com"); user.setId(new UserId(UUID.randomUUID())); - service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user, System.currentTimeMillis()); + service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), System.currentTimeMillis()); ObjectNode commentNode = JacksonUtil.newObjectNode(); commentNode.put("subtype", "ASSIGN"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index ba37737b84..7bfe76e753 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -24,6 +24,7 @@ import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -78,9 +79,8 @@ public abstract class AbstractEntityService { log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); cleanUpRelatedData(tenantId, entityId); - if (EntityType.USER.equals(entityId.getEntityType())) { -// housekeeperService.submitTask(HousekeeperTask.unassignAlarms(tenantId, entityId)); -// unassignDeletedUserAlarms(tenantId, (User) event.getEntity(), event.getTs()); + if (entityId.getEntityType() == EntityType.USER) { + housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java similarity index 59% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java rename to dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index 55b164df52..876a8f736b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -13,17 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper; +package org.thingsboard.server.dao.housekeeper.data; -import com.google.common.util.concurrent.ListenableFuture; +import lombok.Getter; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.TenantId; -import java.util.List; +@Getter +public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { -public interface HouseKeeperService { + private final String userTitle; + private final long ts; - ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + protected AlarmsUnassignHousekeeperTask(User user) { + super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); + this.userTitle = user.getTitle(); + this.ts = System.currentTimeMillis(); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 7ab415fe7b..726e07a790 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -15,27 +15,28 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UserId; import java.io.Serializable; /* * on start, read the retry queue and put the messages back to main queue (save offset) * */ -@Data +@Getter public class HousekeeperTask implements Serializable { private final TenantId tenantId; private final EntityId entityId; private final HousekeeperTaskType taskType; - // maybe we should not delete relations asynchronously -// public static HousekeeperTask deleteRelations(TenantId tenantId, EntityId entityId) { -// return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_RELATIONS); -// } + protected HousekeeperTask(TenantId tenantId, EntityId entityId, HousekeeperTaskType taskType) { + this.tenantId = tenantId; + this.entityId = entityId; + this.taskType = taskType; + } public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES); @@ -49,8 +50,8 @@ public class HousekeeperTask implements Serializable { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); } - public static HousekeeperTask unassignAlarms(TenantId tenantId, UserId userId) { - return new HousekeeperTask(tenantId, userId, HousekeeperTaskType.UNASSIGN_ALARMS); + public static HousekeeperTask unassignAlarms(User user) { + return new AlarmsUnassignHousekeeperTask(user); } public static HousekeeperTask deleteEntityAlarms(TenantId tenantId, EntityId entityId) { From c0f9ef8a6dc6686139cf246153ddd5e138d72125 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 9 Feb 2024 15:09:00 +0200 Subject: [PATCH 010/410] new entity for queue statistics: added cleanup for tenant, added tests, updated UI --- .../dashboards/rule_engine_statistics.json | 17 +-- .../main/data/upgrade/3.6.2/schema_update.sql | 5 +- .../server/controller/BaseController.java | 13 -- .../service/security/permission/Resource.java | 3 +- .../controller/BaseQueueControllerTest.java | 1 + .../controller/EntityQueryControllerTest.java | 47 ++++++ .../server/dao/queue/QueueStatsService.java | 4 +- .../common/data/id/EntityIdFactory.java | 2 + .../server/common/data/queue/QueueStats.java | 6 +- common/proto/src/main/proto/queue.proto | 1 + .../server/dao/asset/BaseAssetService.java | 1 - .../dao/queue/BaseQueueStatsService.java | 21 ++- .../server/dao/queue/QueueStatsDao.java | 4 +- .../validator/QueueStatsDataValidator.java | 40 ++++++ .../dao/sql/query/EntityKeyMapping.java | 5 + .../dao/sql/queue/JpaQueueStatsDao.java | 9 +- .../dao/sql/queue/QueueStatsRepository.java | 11 +- .../server/dao/tenant/TenantServiceImpl.java | 6 + .../dao/service/QueueStatsServiceTest.java | 134 ++++++++++++++++++ ui-ngx/src/assets/dashboard/api_usage.json | 17 +-- 20 files changed, 298 insertions(+), 49 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueStatsDataValidator.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/QueueStatsServiceTest.java diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json index c84faa359e..2a079fcd9d 100644 --- a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json +++ b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json @@ -105,7 +105,7 @@ "_hash": 0.49891007198715376 } ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + "entityAliasId": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba" } ], "timewindow": { @@ -228,7 +228,7 @@ "_hash": 0.7255162989552142 } ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + "entityAliasId": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba" } ], "timewindow": { @@ -341,7 +341,7 @@ "_hash": 0.2679547062508352 } ], - "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018" + "entityAliasId": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba" } ], "timewindow": { @@ -463,16 +463,13 @@ } }, "entityAliases": { - "140f23dd-e3a0-ed98-6189-03c49d2d8018": { - "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", + "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba": { + "id": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba", "alias": "TbServiceQueues", "filter": { - "type": "assetType", + "type": "entityType", "resolveMultiple": true, - "assetNameFilter": "", - "assetTypes": [ - "TbServiceQueue" - ] + "entityType": "QUEUE_STATS" } } }, diff --git a/application/src/main/data/upgrade/3.6.2/schema_update.sql b/application/src/main/data/upgrade/3.6.2/schema_update.sql index 1be58e4408..6e0cfd7d95 100644 --- a/application/src/main/data/upgrade/3.6.2/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.2/schema_update.sql @@ -40,9 +40,10 @@ CREATE TABLE IF NOT EXISTS queue_stats ( CONSTRAINT queue_stats_name_unq_key UNIQUE (tenant_id, queue_name, service_id)); INSERT INTO queue_stats - SELECT id, created_time, tenant_id, split_part(name, '_', 1) AS queue_name, split_part(name, '_', 2) AS service_id + SELECT id, created_time, tenant_id, substring(name FROM 1 FOR position('_' IN name) - 1) AS queue_name, + substring(name FROM position('_' IN name) + 1) AS service_id FROM asset - WHERE type = 'TbServiceQueue'; + WHERE type = 'TbServiceQueue' and name LIKE '%\_%'; DELETE FROM asset WHERE type='TbServiceQueue'; DELETE FROM asset_profile WHERE name ='TbServiceQueue'; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 5f406eed29..c40760dda3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -78,7 +78,6 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.QueueId; -import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -97,7 +96,6 @@ import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -125,7 +123,6 @@ import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rpc.RpcService; @@ -310,9 +307,6 @@ public abstract class BaseController { @Autowired protected QueueService queueService; - @Autowired - protected QueueStatsService queueStatsService; - @Autowired protected EntitiesVersionControlService vcService; @@ -606,9 +600,6 @@ public abstract class BaseController { case QUEUE: checkQueueId(new QueueId(entityId.getId()), operation); return; - case QUEUE_STATS: - checkQueueStatsId(new QueueStatsId(entityId.getId()), operation); - return; default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } @@ -785,10 +776,6 @@ public abstract class BaseController { return queue; } - protected QueueStats checkQueueStatsId(QueueStatsId queueStatsId, Operation operation) throws ThingsboardException { - return checkEntityId(queueStatsId, queueStatsService::findQueueStatsById, operation); - } - protected I emptyId(EntityType entityType) { return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 770745f738..5aa869f2a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -46,8 +46,7 @@ public enum Resource { QUEUE(EntityType.QUEUE), VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, - EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - QUEUE_STATS(EntityType.QUEUE_STATS); + EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE); private final Set entityTypes; diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 87688a797f..b0a6db944c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -179,6 +179,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest { List queueStatsList = queueStatsService.findByTenantId(tenantId); assertThat(queueStatsList).hasSize(1); QueueStats queueStats = queueStatsList.get(0); + assertThat(queueStats.getQueueName()).isEqualTo(queue.getName()); ArgumentCaptor ttlCaptor = ArgumentCaptor.forClass(Long.class); verify(timeseriesDao).save(eq(tenantId), eq(queueStats.getId()), argThat(tsKvEntry -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 67ad2cbe55..577feeb9fb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -22,11 +22,13 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; @@ -51,10 +53,13 @@ import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -69,6 +74,9 @@ public class EntityQueryControllerTest extends AbstractControllerTest { private Tenant savedTenant; private User tenantAdmin; + @Autowired + private QueueStatsService queueStatsService; + @Before public void beforeTest() throws Exception { loginSysAdmin(); @@ -593,4 +601,43 @@ public class EntityQueryControllerTest extends AbstractControllerTest { assertThat(getErrorMessage(result)).contains("Invalid").contains("sort property"); } + @Test + public void testFindQueueStatsEntitiesByQuery() throws Exception { + List queueStatsList = new ArrayList<>(); + for (int i = 0; i < 97; i++) { + QueueStats queueStats = new QueueStats(); + queueStats.setQueueName(StringUtils.randomAlphabetic(5)); + queueStats.setServiceId(StringUtils.randomAlphabetic(5)); + queueStats.setTenantId(savedTenant.getTenantId()); + queueStatsList.add(queueStatsService.save(savedTenant.getId(), queueStats)); + Thread.sleep(1); + } + + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(EntityType.QUEUE_STATS); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder( + new EntityKey(EntityKeyType.ENTITY_FIELD, "queueName"), EntityDataSortOrder.Direction.ASC + ); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "queueName"), + new EntityKey(EntityKeyType.ENTITY_FIELD, "serviceId")); + + EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null); + + PageData data = + doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { + }); + + Assert.assertEquals(97, data.getTotalElements()); + Assert.assertEquals(10, data.getTotalPages()); + Assert.assertTrue(data.hasNext()); + Assert.assertEquals(10, data.getData().size()); + + EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter); + + Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + Assert.assertEquals(97, count.longValue()); + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java index e7166c4fd6..9ab67c73a7 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java @@ -28,8 +28,10 @@ public interface QueueStatsService extends EntityDaoService { QueueStats findQueueStatsById(TenantId tenantId, QueueStatsId queueStatsId); - QueueStats findByTenantIdAndNameAndServiceId(TenantId tenantId, String queueStatsName, String serviceId); + QueueStats findByTenantIdAndNameAndServiceId(TenantId tenantId, String queueName, String serviceId); List findByTenantId(TenantId tenantId); + void deleteByTenantId(TenantId tenantId); + } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index eb8b337c7d..7a9e4388f7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -103,6 +103,8 @@ public class EntityIdFactory { return new NotificationTemplateId(uuid); case NOTIFICATION: return new NotificationId(uuid); + case QUEUE_STATS: + return new QueueStatsId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java index b6224d4167..04d50dfe6a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java @@ -16,13 +16,15 @@ package org.thingsboard.server.common.data.queue; import lombok.Data; -import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.QueueStatsId; +@EqualsAndHashCode(callSuper = true) @Data -public class QueueStats extends BaseDataWithAdditionalInfo implements HasTenantId { +public class QueueStats extends BaseData implements HasTenantId { private TenantId tenantId; private String queueName; private String serviceId; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4da2883291..a332c1858b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -54,6 +54,7 @@ enum EntityTypeProto { NOTIFICATION_REQUEST = 31; NOTIFICATION = 32; NOTIFICATION_RULE = 33; + QUEUE_STATS = 34; } /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 7220a3acdb..e88618afba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -75,7 +75,6 @@ public class BaseAssetService extends AbstractCachedEntityService queueStatsValidator; @Override public QueueStats save(TenantId tenantId, QueueStats queueStats) { + log.trace("Executing save [{}]", queueStats); + queueStatsValidator.validate(queueStats, QueueStats::getTenantId); return queueStatsDao.save(tenantId, queueStats); } @@ -55,15 +60,23 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu @Override public QueueStats findByTenantIdAndNameAndServiceId(TenantId tenantId, String queueName, String serviceId) { log.trace("Executing findByTenantIdAndNameAndServiceId, tenantId: [{}], queueName: [{}], serviceId: [{}]", tenantId, queueName, serviceId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); return queueStatsDao.findByTenantIdQueueNameAndServiceId(tenantId, queueName, serviceId); } @Override public List findByTenantId(TenantId tenantId) { log.trace("Executing findByTenantId, tenantId: [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); return queueStatsDao.findByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + log.trace("Executing deleteDevicesByTenantId, tenantId [{}]", tenantId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + queueStatsDao.deleteByTenantId(tenantId); + } @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { @@ -72,6 +85,6 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu @Override public EntityType getEntityType() { - return null; + return EntityType.QUEUE_STATS; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java index fc83891036..1c3db8bb54 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java @@ -23,8 +23,10 @@ import java.util.List; public interface QueueStatsDao extends Dao { - QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String name, String serviceId); + QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String queueName, String serviceId); List findByTenantId(TenantId tenantId); + void deleteByTenantId(TenantId tenantId); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueStatsDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueStatsDataValidator.java new file mode 100644 index 0000000000..92134b98d5 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueStatsDataValidator.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class QueueStatsDataValidator extends DataValidator { + + @Override + protected void validateDataImpl(TenantId tenantId, QueueStats queueStats) { + if (queueStats.getTenantId() == null) { + throw new DataValidationException("Tenant id should be specified!."); + } + if (queueStats.getQueueName() == null) { + throw new DataValidationException("Queue name should be specified!."); + } + if (StringUtils.isEmpty(queueStats.getServiceId())) { + throw new DataValidationException("Service id should be specified!."); + } + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index f5874cbd08..43cc3dbc69 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -73,6 +73,8 @@ public class EntityKeyMapping { public static final String PHONE = "phone"; public static final String ADDITIONAL_INFO = "additionalInfo"; public static final String RELATED_PARENT_ID = "parentId"; + public static final String QUEUE_NAME = "queueName"; + public static final String SERVICE_ID = "serviceId"; public static final List typedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, ADDITIONAL_INFO); public static final List widgetEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME); @@ -104,6 +106,7 @@ public class EntityKeyMapping { allowedEntityFieldMap.put(EntityType.API_USAGE_STATE, apiUsageStateEntityFields); allowedEntityFieldMap.put(EntityType.DEVICE_PROFILE, Set.of(CREATED_TIME, NAME, TYPE)); allowedEntityFieldMap.put(EntityType.ASSET_PROFILE, Set.of(CREATED_TIME, NAME)); + allowedEntityFieldMap.put(EntityType.QUEUE_STATS, new HashSet<>(Arrays.asList(CREATED_TIME, QUEUE_NAME, SERVICE_ID))); entityFieldColumnMap.put(CREATED_TIME, ModelConstants.CREATED_TIME_PROPERTY); entityFieldColumnMap.put(ENTITY_TYPE, ModelConstants.ENTITY_TYPE_PROPERTY); @@ -124,6 +127,8 @@ public class EntityKeyMapping { entityFieldColumnMap.put(PHONE, ModelConstants.PHONE_PROPERTY); entityFieldColumnMap.put(ADDITIONAL_INFO, ModelConstants.ADDITIONAL_INFO_PROPERTY); entityFieldColumnMap.put(RELATED_PARENT_ID, "parent_id"); + entityFieldColumnMap.put(QUEUE_NAME, ModelConstants.QUEUE_STATS_QUEUE_NAME_PROPERTY); + entityFieldColumnMap.put(SERVICE_ID, ModelConstants.QUEUE_STATS_SERVICE_ID_PROPERTY); Map contactBasedAliases = new HashMap<>(); contactBasedAliases.put(NAME, TITLE); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index 29853ce611..ac57d54e90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -49,8 +49,8 @@ public class JpaQueueStatsDao extends JpaAbstractDao { - QueueStatsEntity findByTenantIdAndQueueNameAndServiceId(UUID tenantId, String name, String serviceId); + QueueStatsEntity findByTenantIdAndQueueNameAndServiceId(UUID tenantId, String queueName, String serviceId); List findByTenantId(UUID tenantId); + @Transactional + @Modifying + @Query("DELETE FROM QueueStatsEntity t WHERE t.tenantId = :tenantId") + void deleteByTenantId(@Param("tenantId") UUID tenantId); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index d91f469c1b..cfef5d4cd0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -49,6 +49,7 @@ import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTemplateService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rpc.RpcService; import org.thingsboard.server.dao.rule.RuleChainService; @@ -131,6 +132,10 @@ public class TenantServiceImpl extends AbstractCachedEntityService 0); + Assert.assertEquals(queueStats.getTenantId(), savedQueueStats.getTenantId()); + Assert.assertEquals(savedQueueStats.getQueueName(), queueStats.getQueueName()); + + QueueStats retrievedQueueStatsById = queueStatsService.findQueueStatsById(tenantId, savedQueueStats.getId()); + Assert.assertEquals(retrievedQueueStatsById.getQueueName(), queueName); + + String secondQueueName = StringUtils.randomAlphabetic(8); + queueStats.setQueueName(secondQueueName); + QueueStats savedQueueStats2 = queueStatsService.save(tenantId, queueStats); + QueueStats retrievedQueueStatsById2 = queueStatsService.findQueueStatsById(tenantId, savedQueueStats2.getId()); + Assert.assertEquals(retrievedQueueStatsById2.getQueueName(), secondQueueName); + + List queueStatsList = queueStatsService.findByTenantId(tenantId); + Assert.assertEquals(2, queueStatsList.size()); + assertThat(queueStatsList).containsOnly(retrievedQueueStatsById, retrievedQueueStatsById2); + + queueStatsService.deleteByTenantId(tenantId); + QueueStats retrievedQueueStatsAfterDelete = queueStatsService.findQueueStatsById(tenantId, savedQueueStats.getId()); + Assert.assertNull(retrievedQueueStatsAfterDelete); + } + + @Test + public void testSaveWithNullQueueName() { + QueueStats queueStats = new QueueStats(); + queueStats.setTenantId(tenantId); + queueStats.setQueueName(null); + queueStats.setServiceId(StringUtils.randomAlphabetic(8)); + + Assertions.assertThrows(DataValidationException.class, () -> { + queueStatsService.save(tenantId, queueStats); + }); + } + + @Test + public void testSaveWithNullServiceId() { + QueueStats queueStats = new QueueStats(); + queueStats.setTenantId(tenantId); + queueStats.setQueueName(StringUtils.randomAlphabetic(8)); + queueStats.setServiceId(null); + + Assertions.assertThrows(DataValidationException.class, () -> { + queueStatsService.save(tenantId, queueStats); + }); + } + + @Test + public void testFindByTenantIdAndNameAndServiceId() { + QueueStats queueStats = new QueueStats(); + queueStats.setTenantId(tenantId); + queueStats.setQueueName(StringUtils.randomAlphabetic(8)); + queueStats.setServiceId(StringUtils.randomAlphabetic(8)); + QueueStats savedQueueStats = queueStatsService.save(tenantId, queueStats); + + QueueStats queueStats2 = new QueueStats(); + queueStats2.setTenantId(tenantId); + queueStats2.setQueueName(StringUtils.randomAlphabetic(8)); + queueStats2.setServiceId(StringUtils.randomAlphabetic(8)); + queueStatsService.save(tenantId, queueStats2); + + QueueStats retrievedQueueStatsById = queueStatsService.findByTenantIdAndNameAndServiceId(tenantId, queueStats.getQueueName(), queueStats.getServiceId()); + assertThat(retrievedQueueStatsById).isEqualTo(savedQueueStats); + } + +} diff --git a/ui-ngx/src/assets/dashboard/api_usage.json b/ui-ngx/src/assets/dashboard/api_usage.json index b1cea42249..9e0af9556a 100644 --- a/ui-ngx/src/assets/dashboard/api_usage.json +++ b/ui-ngx/src/assets/dashboard/api_usage.json @@ -3729,7 +3729,7 @@ "_hash": 0.49891007198715376 } ], - "entityAliasId": "2e4c97b0-257a-a1b9-690c-141d9bf2ec6f" + "entityAliasId": "1fcc06b0-ba0d-11ee-8765-750128ef3ba9" } ], "timewindow": { @@ -3880,7 +3880,7 @@ "_hash": 0.2679547062508352 } ], - "entityAliasId": "2e4c97b0-257a-a1b9-690c-141d9bf2ec6f" + "entityAliasId": "1fcc06b0-ba0d-11ee-8765-750128ef3ba9" } ], "timewindow": { @@ -4007,7 +4007,7 @@ "_hash": 0.7255162989552142 } ], - "entityAliasId": "2e4c97b0-257a-a1b9-690c-141d9bf2ec6f" + "entityAliasId": "1fcc06b0-ba0d-11ee-8765-750128ef3ba9" } ], "timewindow": { @@ -4880,16 +4880,13 @@ "resolveMultiple": false } }, - "2e4c97b0-257a-a1b9-690c-141d9bf2ec6f": { - "id": "2e4c97b0-257a-a1b9-690c-141d9bf2ec6f", + "1fcc06b0-ba0d-11ee-8765-750128ef3ba9": { + "id": "1fcc06b0-ba0d-11ee-8765-750128ef3ba9", "alias": "TbServiceQueues", "filter": { - "type": "assetType", + "type": "entityType", "resolveMultiple": true, - "assetNameFilter": "", - "assetTypes": [ - "TbServiceQueue" - ] + "entityType": "QUEUE_STATS" } } }, From aa24da06ea6a2e56bd6e0952f3fee71988a3544c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 9 Feb 2024 16:13:45 +0200 Subject: [PATCH 011/410] TenantIdLoaderTest fix --- .../server/controller/EntityQueryControllerTest.java | 4 ++++ .../org/thingsboard/rule/engine/api/TbContext.java | 3 +++ .../thingsboard/rule/engine/util/TenantIdLoader.java | 4 ++++ .../rule/engine/util/TenantIdLoaderTest.java | 10 ++++++++++ 4 files changed, 21 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 577feeb9fb..8de8514e5b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -633,6 +633,10 @@ public class EntityQueryControllerTest extends AbstractControllerTest { Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); + data.getData().forEach(entityData -> { + assertThat(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("queueName")).asString().isNotBlank(); + assertThat(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("serviceId")).asString().isNotBlank(); + }); EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 59f5b128a7..3d74b70447 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -69,6 +69,7 @@ import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTemplateService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; @@ -314,6 +315,8 @@ public interface TbContext { QueueService getQueueService(); + QueueStatsService getQueueStatsService(); + ListeningExecutor getMailExecutor(); ListeningExecutor getSmsExecutor(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 0900137b73..152e193ea4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.id.NotificationTemplateId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -141,6 +142,9 @@ public class TenantIdLoader { case NOTIFICATION_RULE: tenantEntity = ctx.getNotificationRuleService().findNotificationRuleById(ctxTenantId, new NotificationRuleId(id)); break; + case QUEUE_STATS: + tenantEntity = ctx.getQueueStatsService().findQueueStatsById(ctxTenantId, new QueueStatsId(id)); + break; default: throw new RuntimeException("Unexpected entity type: " + entityId.getEntityType()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index 5648b02454..e06be15fe6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -56,6 +56,7 @@ import org.thingsboard.server.common.data.notification.rule.NotificationRule; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleNode; @@ -73,6 +74,7 @@ import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTemplateService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.user.UserService; @@ -135,6 +137,8 @@ public class TenantIdLoaderTest { private NotificationRequestService notificationRequestService; @Mock private NotificationRuleService notificationRuleService; + @Mock + private QueueStatsService queueStatsService; private TenantId tenantId; private TenantProfileId tenantProfileId; @@ -352,6 +356,12 @@ public class TenantIdLoaderTest { when(ctx.getNotificationRuleService()).thenReturn(notificationRuleService); doReturn(notificationRule).when(notificationRuleService).findNotificationRuleById(eq(tenantId), any()); break; + case QUEUE_STATS: + QueueStats queueStats = new QueueStats(); + queueStats.setTenantId(tenantId); + when(ctx.getQueueStatsService()).thenReturn(queueStatsService); + doReturn(queueStats).when(queueStatsService).findQueueStatsById(eq(tenantId), any()); + break; default: throw new RuntimeException("Unexpected originator EntityType " + entityType); } From 077a5fa49226b48b1efef76f4081dab887d52096 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 9 Feb 2024 16:23:21 +0200 Subject: [PATCH 012/410] added QueueStatsService to DefaultTbContext --- .../org/thingsboard/server/actors/ActorSystemContext.java | 6 ++++++ .../server/actors/ruleChain/DefaultTbContext.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 31d9e477ac..fa717a2613 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -78,6 +78,7 @@ import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTemplateService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; @@ -446,6 +447,11 @@ public class ActorSystemContext { @Getter private QueueService queueService; + @Lazy + @Autowired(required = false) + @Getter + private QueueStatsService queueStatsService; + @Lazy @Autowired(required = false) @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index cc6653f4c5..d34a099d26 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -96,6 +96,7 @@ import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTemplateService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.queue.QueueStatsService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; @@ -738,6 +739,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getQueueService(); } + @Override + public QueueStatsService getQueueStatsService() { + return mainCtx.getQueueStatsService(); + } + @Override public EventLoopGroup getSharedEventLoop() { return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup(); From a018cdf09a807d0237ac74587fca5e62169518de Mon Sep 17 00:00:00 2001 From: rusikv Date: Tue, 13 Feb 2024 13:01:39 +0200 Subject: [PATCH 013/410] UI: QUEUE_STATS for API usage --- .../dashboards/rule_engine_statistics.json | 94 ++++++++++++++++++- ui-ngx/src/app/core/http/entity.service.ts | 5 + .../profile/asset-profile.component.html | 2 +- .../profile/asset-profile.component.ts | 4 +- .../asset-profiles-table-config.resolver.ts | 9 +- ui-ngx/src/app/shared/models/asset.models.ts | 2 - .../app/shared/models/entity-type.models.ts | 8 ++ ui-ngx/src/app/shared/models/entity.models.ts | 10 ++ .../app/shared/models/query/query.models.ts | 3 + ui-ngx/src/assets/dashboard/api_usage.json | 94 ++++++++++++++++++- .../assets/locale/locale.constant-en_US.json | 6 +- 11 files changed, 215 insertions(+), 22 deletions(-) diff --git a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json index 2a079fcd9d..576b1432ff 100644 --- a/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json +++ b/application/src/main/data/json/demo/dashboards/rule_engine_statistics.json @@ -13,6 +13,7 @@ "datasources": [ { "type": "entity", + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018", "dataKeys": [ { "name": "successfulMsgs", @@ -105,7 +106,24 @@ "_hash": 0.49891007198715376 } ], - "entityAliasId": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba" + "latestDataKeys": [ + { + "name": "queueName", + "type": "entityField", + "label": "Queue name", + "color": "#ffc107", + "settings": {}, + "_hash": 0.019706324241253403 + }, + { + "name": "serviceId", + "type": "entityField", + "label": "Service Id", + "color": "#607d8b", + "settings": {}, + "_hash": 0.6439850190675356 + } + ] } ], "timewindow": { @@ -190,6 +208,7 @@ "datasources": [ { "type": "entity", + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018", "dataKeys": [ { "name": "ruleEngineException", @@ -228,7 +247,54 @@ "_hash": 0.7255162989552142 } ], - "entityAliasId": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba" + "latestDataKeys": [ + { + "name": "queueName", + "type": "entityField", + "label": "Queue name", + "color": "#ffc107", + "settings": { + "show": false, + "order": null, + "useCellStyleFunction": false, + "cellStyleFunction": "", + "useCellContentFunction": false, + "cellContentFunction": "", + "defaultColumnVisibility": "visible", + "columnSelectionToDisplay": "enabled" + }, + "_hash": 0.15709042234841886, + "aggregationType": null, + "units": null, + "decimals": null, + "funcBody": null, + "usePostProcessing": null, + "postFuncBody": null + }, + { + "name": "serviceId", + "type": "entityField", + "label": "Service Id", + "color": "#607d8b", + "settings": { + "show": false, + "order": null, + "useCellStyleFunction": false, + "cellStyleFunction": "", + "useCellContentFunction": false, + "cellContentFunction": "", + "defaultColumnVisibility": "visible", + "columnSelectionToDisplay": "enabled" + }, + "_hash": 0.13037127418736705, + "aggregationType": null, + "units": null, + "decimals": null, + "funcBody": null, + "usePostProcessing": null, + "postFuncBody": null + } + ] } ], "timewindow": { @@ -279,6 +345,7 @@ "datasources": [ { "type": "entity", + "entityAliasId": "140f23dd-e3a0-ed98-6189-03c49d2d8018", "dataKeys": [ { "name": "timeoutMsgs", @@ -341,7 +408,24 @@ "_hash": 0.2679547062508352 } ], - "entityAliasId": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba" + "latestDataKeys": [ + { + "name": "queueName", + "type": "entityField", + "label": "Queue name", + "color": "#f44336", + "settings": {}, + "_hash": 0.009348067096302426 + }, + { + "name": "serviceId", + "type": "entityField", + "label": "Service Id", + "color": "#ffc107", + "settings": {}, + "_hash": 0.4586005983243109 + } + ] } ], "timewindow": { @@ -463,8 +547,8 @@ } }, "entityAliases": { - "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba": { - "id": "26f9d890-9611-acd4-1cdb-e0a77fd0e7ba", + "140f23dd-e3a0-ed98-6189-03c49d2d8018": { + "id": "140f23dd-e3a0-ed98-6189-03c49d2d8018", "alias": "TbServiceQueues", "filter": { "type": "entityType", diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index b29fd6c9a0..ce1c26f17d 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -716,6 +716,7 @@ export class EntityService { entityTypes.push(EntityType.CUSTOMER); entityTypes.push(EntityType.USER); entityTypes.push(EntityType.DASHBOARD); + entityTypes.push(EntityType.QUEUE_STATS); if (authState.edgesSupportEnabled) { entityTypes.push(EntityType.EDGE); } @@ -795,6 +796,10 @@ export class EntityService { case EntityType.API_USAGE_STATE: entityFieldKeys.push(entityFields.name.keyName); break; + case EntityType.QUEUE_STATS: + entityFieldKeys.push(entityFields.queueName.keyName); + entityFieldKeys.push(entityFields.serviceId.keyName); + break; } return query ? entityFieldKeys.filter((entityField) => entityField.toLowerCase().indexOf(query) === 0) : entityFieldKeys; } diff --git a/ui-ngx/src/app/modules/home/components/profile/asset-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/asset-profile.component.html index 5bf94d5c86..67f3bcdf13 100644 --- a/ui-ngx/src/app/modules/home/components/profile/asset-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/asset-profile.component.html @@ -31,7 +31,7 @@