From 86b5378d592de789afff096e387bb328389c48a7 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 27 Jan 2025 15:16:50 +0200 Subject: [PATCH 01/51] EDQS (#3196) * Experiments with CSV * CSV Loader v1 * EDQ tests * Volatile variables instead of final * Improvements * updated loader with new entities * Fix double memory usage issue * Basic data structures and load * Minor improvements * Snappy + Large String reuse * added EntityFields classes for each entity * Basic implementation * Minor improvements to KeyFilters * implemented RepositoryUtils.checkKeyFilters * Generic query implementation * New structure * Refactoring and few processors implementation * extended DeviceData with shared/client attributes and device profile * Minor refactoring of attribute scopes * DeviceTypeFilter support * Strong types of fields for each entity data class * DeviceType and AssetType filters * EntityView and Edge queries * Relations Query * Relation Query Implementation * Update EDQS module version * Sync with EDQS via Kafka * EDQS: major refactoring * EDQS API requests via Kafka * EDQS: full sync with the database * Refactoring for EDQS sync * EDQS: major refactoring and new features * EDQS refactoring, count query support, fix tests * EDQS: refactoring for query processors * Fix EDQS pom version * Cleanup edqs.yml * EDQS: tenant partitioning strategy; refactoring * EDQS: latest events queue * EDQS: support for monolith setup; RocksDB; other improvements * EDQS: merge sync and events topics, introduce state topic * EDQS: dynamic repartitioning * implemented entity data query filters for edqs * EdqsEntityQueryControllerTest - use in-memory queue * edqs-filter fixes, added test * EDQS: blob entity support * EdqsEntityQueryControllerTest - use in-memory queue * Use DummyEdqsService when disabled * Fixes for EDQS * Refactoring for EDQS tests * Fix edqs requests partitioning * EDQS: Fix for attributes handling * Fix attributes saving in EntityServiceTest * EDQS: refactoring, fixes * Minor refactoring for query processor * added ownerName/ownerType support * fixed relation query processor * fixed EntityServiceTest * refactoring * added support for parentId for relation query result * Get rid of EntityNameFetcher * Add fixme for relation query processor * db restore with select all edqs fields * fixed entity deletion * fixed FieldUtils with new EntityFields * dao method renamed * EDQS: instance groups with same partitions; automatic sync; multiple fixes * Refactoring for EDQS sync * EDQS: refactoring * Fix startup with Kafka * fixed EntityQueryControllerTest * fixed EdqsEntityServiceTest * Separate queue admin for EDQS request template * Implement new EDQS partitioning strategy * EDQS: multiple fixes and refactoring * Add mock EdqsRocksDb beans to tests * added edqs stats for inmemory/grafana * fixed filter tests * Update todos * Refactoring for QueueConfig * Improvements and refactoring for EDQS consumers * implemented TODOs * test fixes * Consume state topic up to end offsets * edqs stats refactoring * EDQS: cleanup on partitions removal; refactoring * EDQS: minor refactoring * EDQS: remove CSV loader --------- Co-authored-by: Andrii Shvaika Co-authored-by: dashevchenko --- application/pom.xml | 4 + .../controller/EntityQueryController.java | 11 + .../edge/EdgeEventSourcingListener.java | 5 + .../service/edqs/DefaultEdqsService.java | 340 ++++++++++ .../server/service/edqs/EdqsDataLoader.java | 539 ++++++++++++++++ .../server/service/edqs/EdqsListener.java | 61 ++ .../server/service/edqs/EdqsSyncService.java | 275 ++++++++ .../service/edqs/KafkaEdqsSyncService.java | 47 ++ .../service/edqs/LocalEdqsSyncService.java | 35 + .../entitiy/EntityStateSourcingListener.java | 5 + .../queue/DefaultTbCoreConsumerService.java | 27 +- .../queue/DefaultTbEdgeConsumerService.java | 16 +- .../DefaultTbRuleEngineConsumerService.java | 2 +- .../TbRuleEngineQueueConsumerManager.java | 5 +- .../state/DefaultDeviceStateService.java | 31 +- .../sync/tenant/TenantExportService.java | 224 +++++++ .../transport/TbCoreTransportApiService.java | 3 +- .../service/ws/DefaultWebSocketService.java | 1 + .../src/main/resources/thingsboard.yml | 23 + .../EdqsEntityQueryControllerTest.java | 67 ++ .../controller/EntityQueryControllerTest.java | 167 +++-- .../discovery/HashPartitionServiceTest.java | 16 +- .../entitiy/EdqsEntityServiceTest.java | 72 +++ .../service/entitiy}/EntityServiceTest.java | 609 ++++++++++-------- .../TbRuleEngineQueueConsumerManagerTest.java | 11 +- .../state/DefaultDeviceStateServiceTest.java | 2 +- .../resources/application-test.properties | 5 +- .../src/test/resources/logback-test.xml | 2 +- .../server/queue/TbQueueRequestTemplate.java | 2 + .../server/queue/TbQueueResponseTemplate.java | 10 +- .../server/common/data/ObjectType.java | 102 +++ .../server/common/data/alarm/AlarmType.java | 27 + .../server/common/data/edqs/AttributeKv.java | 67 ++ .../server/common/data/edqs/EdqsEvent.java | 34 + .../common/data/edqs/EdqsEventType.java | 21 + .../server/common/data/edqs/EdqsObject.java | 28 + .../common/data/edqs/EdqsSyncRequest.java | 24 + .../server/common/data/edqs/Entity.java | 62 ++ .../server/common/data/edqs/LatestTsKv.java | 62 ++ .../common/data/edqs/ToCoreEdqsMsg.java | 32 + .../common/data/edqs/ToCoreEdqsRequest.java | 38 ++ .../edqs/fields/AbstractEntityFields.java | 65 ++ .../data/edqs/fields/ApiUsageStateFields.java | 58 ++ .../common/data/edqs/fields/AssetFields.java | 58 ++ .../data/edqs/fields/AssetProfileFields.java | 35 + .../data/edqs/fields/CustomerFields.java | 55 ++ .../data/edqs/fields/DashboardFields.java | 33 + .../common/data/edqs/fields/DeviceFields.java | 58 ++ .../data/edqs/fields/DeviceProfileFields.java | 38 ++ .../common/data/edqs/fields/EdgeFields.java | 43 ++ .../common/data/edqs/fields/EntityFields.java | 171 +++++ .../data/edqs/fields/EntityIdFields.java | 36 ++ .../data/edqs/fields/EntityViewFields.java | 30 + .../common/data/edqs/fields/FieldsUtil.java | 298 +++++++++ .../data/edqs/fields/GenericFields.java | 41 ++ .../data/edqs/fields/ProfileAwareFields.java | 26 + .../data/edqs/fields/QueueStatsFields.java | 42 ++ .../data/edqs/fields/RuleChainFields.java | 38 ++ .../data/edqs/fields/RuleNodeFields.java | 43 ++ .../common/data/edqs/fields/TenantFields.java | 62 ++ .../data/edqs/fields/TenantProfileFields.java | 36 ++ .../common/data/edqs/fields/UserFields.java | 48 ++ .../data/edqs/fields/WidgetTypeFields.java | 30 + .../data/edqs/fields/WidgetsBundleFields.java | 30 + .../common/data/edqs/query/EdqsRequest.java | 38 ++ .../common/data/edqs/query/EdqsResponse.java | 35 + .../common/data/edqs/query/QueryResult.java | 41 ++ .../common/data/id/UserAuthSettingsId.java | 6 +- .../data/page/BasePageDataIterable.java | 9 +- .../common/data/page/PageDataIterable.java | 5 + .../common/data/permission/QueryContext.java | 14 +- .../common/data/query/DynamicValue.java | 2 - .../common/data/query/EntityCountQuery.java | 2 + .../server/common/data/query/EntityData.java | 15 +- .../server/common/data/queue/QueueConfig.java | 12 + .../server/common/data/queue/QueueStats.java | 8 +- .../common/data/relation/EntityRelation.java | 15 +- .../common/data/util/CollectionsUtil.java | 8 + common/edqs/pom.xml | 97 +++ .../server/edqs/data/ApiUsageStateData.java | 46 ++ .../server/edqs/data/AssetData.java | 36 ++ .../server/edqs/data/BaseEntityData.java | 180 ++++++ .../server/edqs/data/CustomerData.java | 62 ++ .../server/edqs/data/DeviceData.java | 86 +++ .../server/edqs/data/EntityData.java | 65 ++ .../server/edqs/data/EntityGroupData.java | 58 ++ .../server/edqs/data/EntityProfileData.java | 39 ++ .../server/edqs/data/GenericData.java | 38 ++ .../server/edqs/data/ProfileAwareData.java | 28 + .../server/edqs/data/RelationData.java | 26 + .../server/edqs/data/RelationInfo.java | 26 + .../server/edqs/data/RelationsRepo.java | 62 ++ .../server/edqs/data/TenantData.java | 34 + .../edqs/data/dp/AbstractDataPoint.java | 56 ++ .../server/edqs/data/dp/BoolDataPoint.java | 46 ++ .../edqs/data/dp/CompressedJsonDataPoint.java | 31 + .../data/dp/CompressedStringDataPoint.java | 64 ++ .../server/edqs/data/dp/DataPoint.java | 40 ++ .../server/edqs/data/dp/DoubleDataPoint.java | 46 ++ .../server/edqs/data/dp/JsonDataPoint.java | 47 ++ .../server/edqs/data/dp/LongDataPoint.java | 50 ++ .../server/edqs/data/dp/StringDataPoint.java | 51 ++ .../server/edqs/load/TenantRepoLoader.java | 144 +++++ .../server/edqs/processor/EdqsConverter.java | 183 ++++++ .../server/edqs/processor/EdqsProcessor.java | 266 ++++++++ .../server/edqs/processor/EdqsProducer.java | 83 +++ .../server/edqs/query/DataKey.java | 22 + .../server/edqs/query/EdqsCountQuery.java | 30 + .../server/edqs/query/EdqsDataQuery.java | 59 ++ .../server/edqs/query/EdqsFilter.java | 23 + .../server/edqs/query/EdqsQuery.java | 30 + .../server/edqs/query/SortableEntityData.java | 40 ++ .../AbstractEntityGroupQueryProcessor.java | 75 +++ ...stractEntityProfileNameQueryProcessor.java | 52 ++ .../AbstractEntityProfileQueryProcessor.java | 62 ++ .../AbstractEntitySearchQueryProcessor.java | 66 ++ .../processor/AbstractQueryProcessor.java | 129 ++++ .../AbstractRelationQueryProcessor.java | 260 ++++++++ .../AbstractSimpleQueryProcessor.java | 99 +++ ...bstractSingleEntityTypeQueryProcessor.java | 172 +++++ .../ApiUsageStateQueryProcessor.java | 88 +++ .../processor/AssetSearchQueryProcessor.java | 59 ++ .../processor/AssetTypeQueryProcessor.java | 47 ++ .../query/processor/CombinedPermissions.java | 25 + .../processor/DeviceSearchQueryProcessor.java | 59 ++ .../processor/DeviceTypeQueryProcessor.java | 47 ++ .../processor/EdgeTypeQueryProcessor.java | 42 ++ .../EdgeTypeSearchQueryProcessor.java | 44 ++ .../EntitiesByGroupNameQueryProcessor.java | 121 ++++ .../EntitiesByGroupQueryProcessor.java | 96 +++ .../EntityGroupListQueryProcessor.java | 84 +++ .../EntityGroupNameQueryProcessor.java | 83 +++ .../processor/EntityListQueryProcessor.java | 94 +++ .../processor/EntityNameQueryProcessor.java | 41 ++ .../query/processor/EntityQueryProcessor.java | 28 + .../EntityQueryProcessorFactory.java | 50 ++ .../processor/EntityTypeQueryProcessor.java | 35 + .../EntityViewSearchQueryProcessor.java | 44 ++ .../EntityViewTypeQueryProcessor.java | 42 ++ .../query/processor/GroupPermissions.java | 29 + .../edqs/query/processor/Permissions.java | 24 + .../processor/RelationQueryPermissions.java | 33 + .../processor/RelationQueryProcessor.java | 84 +++ .../SchedulerEventQueryProcessor.java | 37 ++ .../processor/SingleEntityQueryProcessor.java | 87 +++ .../StateEntityOwnerQueryProcessor.java | 91 +++ .../server/edqs/repo/EdqRepository.java | 44 ++ .../edqs/repo/InMemoryEdqRepository.java | 91 +++ .../server/edqs/repo/KeyDictionary.java | 40 ++ .../server/edqs/repo/TbBytePool.java | 39 ++ .../server/edqs/repo/TbStringPool.java | 37 ++ .../server/edqs/repo/TenantRepo.java | 584 +++++++++++++++++ .../server/edqs/state/EdqsStateService.java | 32 + .../edqs/state/KafkaEdqsStateService.java | 187 ++++++ .../edqs/state/LocalEdqsStateService.java | 81 +++ .../server/edqs/stats/EdqsStatsService.java | 91 +++ .../edqs/util/EdqsPartitionService.java | 40 ++ .../server/edqs/util/EdqsRocksDb.java | 51 ++ .../server/edqs/util/RepositoryUtils.java | 424 ++++++++++++ .../server/edqs/util/TbRocksDb.java | 68 ++ .../server/edqs/util/VersionsStore.java | 48 ++ .../server/common/msg/edqs/EdqsService.java | 47 ++ .../server/common/msg/queue/ServiceType.java | 3 +- .../common/msg/queue/TopicPartitionInfo.java | 22 +- common/pom.xml | 1 + common/proto/src/main/proto/queue.proto | 57 ++ common/queue/pom.xml | 4 + .../AbstractTbQueueConsumerTemplate.java | 12 +- .../common/DefaultTbQueueRequestTemplate.java | 20 +- .../DefaultTbQueueResponseTemplate.java | 14 +- .../consumer/MainQueueConsumerManager.java | 18 +- .../queue/common/consumer}/QueueEvent.java | 2 +- .../consumer}/TbQueueConsumerManagerTask.java | 2 +- .../common/consumer}/TbQueueConsumerTask.java | 12 +- .../DefaultTbServiceInfoProvider.java | 10 + .../queue/discovery/HashPartitionService.java | 76 ++- .../queue/discovery/PartitionService.java | 6 +- .../queue/discovery/ZkDiscoveryService.java | 3 + .../discovery/event/PartitionChangeEvent.java | 19 +- .../server/queue/edqs/EdqsComponent.java | 29 + .../server/queue/edqs/EdqsConfig.java | 55 ++ .../server/queue/edqs/EdqsQueue.java | 36 ++ .../server/queue/edqs/EdqsQueueFactory.java | 35 + .../queue/edqs/InMemoryEdqsComponent.java | 26 + .../queue/edqs/InMemoryEdqsQueueFactory.java | 77 +++ .../server/queue/edqs/KafkaEdqsComponent.java | 28 + .../queue/edqs/KafkaEdqsQueueFactory.java | 122 ++++ .../queue/environment/DistributedLock.java | 24 + .../environment/DistributedLockService.java | 22 + .../DummyDistributedLockService.java | 53 ++ .../environment/ZkDistributedLockService.java | 62 ++ .../server/queue/kafka/KafkaTbQueueMsg.java | 8 +- .../server/queue/kafka/TbKafkaAdmin.java | 8 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 89 ++- .../queue/kafka/TbKafkaProducerTemplate.java | 22 +- .../queue/kafka/TbKafkaTopicConfigs.java | 15 + .../provider/EdqsClientQueueFactory.java | 31 + .../InMemoryMonolithQueueFactory.java | 49 +- .../provider/KafkaMonolithQueueFactory.java | 50 +- .../provider/KafkaTbCoreQueueFactory.java | 48 ++ .../KafkaTbRuleEngineQueueFactory.java | 19 + .../queue/provider/TbCoreQueueFactory.java | 2 +- .../provider/TbQueueProducerProvider.java | 2 +- .../provider/TbRuleEngineQueueFactory.java | 2 +- .../server/queue/util/PropertyUtils.java | 3 +- .../DefaultTbQueueRequestTemplateTest.java | 8 +- .../common/stats/DummyMessagesStats.java | 54 ++ .../server/common/stats/StatsType.java | 3 +- .../thingsboard/common/util/JacksonUtil.java | 4 + .../DefaultClusterVersionControlService.java | 2 +- .../java/org/thingsboard/server/dao/Dao.java | 12 +- .../org/thingsboard/server/dao/DaoUtil.java | 5 + .../server/dao/TenantEntityDao.java | 18 +- .../server/dao/asset/AssetDao.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 7 +- .../server/dao/asset/BaseAssetService.java | 2 +- .../server/dao/attributes/AttributesDao.java | 5 + .../dao/attributes/BaseAttributesService.java | 45 +- .../attributes/CachedAttributesService.java | 20 +- .../server/dao/customer/CustomerDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 2 +- .../dao/dashboard/DashboardServiceImpl.java | 2 +- .../server/dao/device/DeviceDao.java | 2 +- .../dao/device/DeviceProfileServiceImpl.java | 7 +- .../dao/dictionary/KeyDictionaryDao.java | 5 + .../thingsboard/server/dao/edge/EdgeDao.java | 2 +- .../server/dao/entity/BaseEntityService.java | 73 +++ .../server/dao/entity/EntityDaoRegistry.java | 31 +- .../dao/entityview/EntityViewServiceImpl.java | 2 +- .../dao/eventsourcing/SaveEntityEvent.java | 1 + .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/AlarmTypeCompositeKey.java | 33 + .../server/dao/model/sql/AlarmTypeEntity.java | 65 ++ .../model/sqlts/latest/TsKvLatestEntity.java | 9 + .../notification/NotificationTargetDao.java | 2 +- .../server/dao/ota/OtaPackageDao.java | 2 + .../dao/queue/BaseQueueStatsService.java | 8 +- .../server/dao/relation/RelationDao.java | 4 + .../server/dao/rule/BaseRuleChainService.java | 12 +- .../server/dao/rule/RuleChainDao.java | 2 +- .../dao/sql/alarm/AlarmCommentRepository.java | 6 +- .../server/dao/sql/alarm/AlarmRepository.java | 2 + .../dao/sql/alarm/AlarmTypeRepository.java | 30 + .../dao/sql/alarm/EntityAlarmRepository.java | 4 + .../dao/sql/alarm/JpaAlarmCommentDao.java | 15 +- .../server/dao/sql/alarm/JpaAlarmDao.java | 14 +- .../server/dao/sql/alarm/JpaAlarmTypeDao.java | 46 ++ .../dao/sql/alarm/JpaEntityAlarmDao.java | 46 ++ .../dao/sql/asset/AssetProfileRepository.java | 5 + .../server/dao/sql/asset/AssetRepository.java | 6 + .../server/dao/sql/asset/JpaAssetDao.java | 17 + .../dao/sql/asset/JpaAssetProfileDao.java | 20 +- .../dao/sql/attributes/JpaAttributeDao.java | 10 + .../dao/sql/customer/CustomerRepository.java | 6 + .../dao/sql/customer/JpaCustomerDao.java | 17 + .../sql/dashboard/DashboardRepository.java | 5 + .../dao/sql/dashboard/JpaDashboardDao.java | 17 + .../device/DeviceCredentialsRepository.java | 6 + .../sql/device/DeviceProfileRepository.java | 6 + .../dao/sql/device/DeviceRepository.java | 6 + .../sql/device/JpaDeviceCredentialsDao.java | 16 +- .../server/dao/sql/device/JpaDeviceDao.java | 17 + .../dao/sql/device/JpaDeviceProfileDao.java | 20 +- .../server/dao/sql/edge/EdgeRepository.java | 5 + .../server/dao/sql/edge/JpaEdgeDao.java | 17 + .../sql/entityview/EntityViewRepository.java | 5 + .../dao/sql/entityview/JpaEntityViewDao.java | 22 +- .../notification/JpaNotificationRuleDao.java | 14 +- .../JpaNotificationTargetDao.java | 11 + .../JpaNotificationTemplateDao.java | 14 +- .../server/dao/sql/ota/JpaOtaPackageDao.java | 19 +- .../dao/sql/ota/JpaOtaPackageInfoDao.java | 1 + .../dao/sql/ota/OtaPackageRepository.java | 6 + .../query/DefaultAlarmQueryRepository.java | 12 +- .../query/DefaultEntityQueryRepository.java | 35 +- .../sql/query/DefaultQueryLogComponent.java | 2 +- .../dao/sql/query/DummyEdqsService.java | 64 ++ .../dao/sql/query/EntityKeyMapping.java | 30 +- .../dao/sql/query/QueryLogComponent.java | 2 +- ...QueryContext.java => SqlQueryContext.java} | 9 +- .../server/dao/sql/queue/JpaQueueDao.java | 14 +- .../dao/sql/queue/JpaQueueStatsDao.java | 6 + .../dao/sql/queue/QueueStatsRepository.java | 5 + .../dao/sql/relation/JpaRelationDao.java | 7 + .../dao/sql/relation/RelationRepository.java | 3 + .../dao/sql/resource/JpaTbResourceDao.java | 9 +- .../server/dao/sql/rpc/JpaRpcDao.java | 14 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 17 + .../server/dao/sql/rule/JpaRuleNodeDao.java | 14 +- .../dao/sql/rule/RuleChainRepository.java | 4 + .../dao/sql/rule/RuleNodeRepository.java | 3 + .../sql/settings/AdminSettingsRepository.java | 4 + .../dao/sql/settings/JpaAdminSettingsDao.java | 17 +- .../server/dao/sql/tenant/JpaTenantDao.java | 12 + .../dao/sql/tenant/JpaTenantProfileDao.java | 6 + .../sql/tenant/TenantProfileRepository.java | 5 + .../dao/sql/tenant/TenantRepository.java | 5 + .../usagerecord/ApiUsageStateRepository.java | 10 + .../sql/usagerecord/JpaApiUsageStateDao.java | 19 + .../dao/sql/user/JpaUserAuthSettingsDao.java | 25 +- .../dao/sql/user/JpaUserCredentialsDao.java | 16 +- .../server/dao/sql/user/JpaUserDao.java | 17 + .../dao/sql/user/JpaUserSettingsDao.java | 17 +- .../sql/user/UserAuthSettingsRepository.java | 5 + .../sql/user/UserCredentialsRepository.java | 6 + .../server/dao/sql/user/UserRepository.java | 6 + .../dao/sql/user/UserSettingsRepository.java | 5 + .../dao/sql/widget/JpaWidgetTypeDao.java | 20 +- .../dao/sql/widget/JpaWidgetsBundleDao.java | 22 +- .../dao/sql/widget/WidgetTypeRepository.java | 5 + .../sql/widget/WidgetsBundleRepository.java | 5 + .../widget/WidgetsBundleWidgetRepository.java | 7 + .../dao/sqlts/AbstractSqlTimeseriesDao.java | 2 +- .../CachedRedisSqlTimeseriesLatestDao.java | 9 + .../dao/sqlts/SqlTimeseriesLatestDao.java | 8 + .../sqlts/dictionary/JpaKeyDictionaryDao.java | 8 + .../dictionary/KeyDictionaryRepository.java | 5 + .../sqlts/latest/TsKvLatestRepository.java | 5 + .../server/dao/tenant/TenantDao.java | 5 +- .../dao/timeseries/BaseTimeseriesService.java | 33 +- .../CassandraBaseTimeseriesLatestDao.java | 9 + .../dao/timeseries/TimeseriesLatestDao.java | 6 + .../dao/usagerecord/ApiUsageStateDao.java | 4 +- .../usagerecord/ApiUsageStateServiceImpl.java | 8 + .../thingsboard/server/dao/user/UserDao.java | 2 +- .../dao/service/AbstractServiceTest.java | 3 +- .../dao/service/EntityDaoRegistryTest.java | 31 + .../query/DefaultQueryLogComponentTest.java | 6 +- edqs/pom.xml | 260 ++++++++ .../edqs/DummyQueueRoutingInfoService.java | 33 + .../edqs/DummyTenantRoutingInfoService.java | 30 + .../edqs/ThingsboardEdqsApplication.java | 119 ++++ edqs/src/main/resources/edqs.yml | 364 +++++++++++ edqs/src/main/resources/logback.xml | 38 ++ .../server/edqs/repo/AbstractEDQTest.java | 296 +++++++++ .../edqs/repo/ApiUsageStateFilterTest.java | 106 +++ .../edqs/repo/AssetSearchQueryFilterTest.java | 223 +++++++ .../server/edqs/repo/AssetTypeFilterTest.java | 187 ++++++ .../repo/DeviceSearchQueryFilterTest.java | 241 +++++++ .../edqs/repo/DeviceTypeFilterTest.java | 142 ++++ .../edqs/repo/EdgeSearchQueryFilterTest.java | 167 +++++ .../server/edqs/repo/EdgeTypeFilterTest.java | 177 +++++ .../repo/EntitiesByGroupIdFilterTest.java | 161 +++++ .../repo/EntitiesByGroupNameFilterTest.java | 140 ++++ .../edqs/repo/EntityGroupListFilterTest.java | 189 ++++++ .../edqs/repo/EntityGroupNameFilterTest.java | 186 ++++++ .../edqs/repo/EntityListFilterTest.java | 202 ++++++ .../edqs/repo/EntityNameFilterTest.java | 132 ++++ .../edqs/repo/EntityTypeFilterTest.java | 147 +++++ .../repo/EntityViewSearchQueryFilterTest.java | 221 +++++++ .../edqs/repo/EntityViewTypeFilterTest.java | 177 +++++ .../edqs/repo/RelationsQueryFilterTest.java | 233 +++++++ .../server/edqs/repo/RepositoryUtilsTest.java | 434 +++++++++++++ .../edqs/repo/SchedulerEventFilterTest.java | 129 ++++ .../edqs/repo/SingleEntityFilterTest.java | 176 +++++ .../edqs/repo/StateEntityOwnerFilterTest.java | 81 +++ edqs/src/test/resources/edq-test.properties | 2 + pom.xml | 18 + 358 files changed, 18294 insertions(+), 675 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java create mode 100644 application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java rename {dao/src/test/java/org/thingsboard/server/dao/service => application/src/test/java/org/thingsboard/server/service/entitiy}/EntityServiceTest.java (87%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java rename dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java => common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java (77%) create mode 100644 common/edqs/pom.xml create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java rename {application/src/main/java/org/thingsboard/server/service/queue => common/queue/src/main/java/org/thingsboard/server/queue/common}/consumer/MainQueueConsumerManager.java (95%) rename {application/src/main/java/org/thingsboard/server/service/queue/ruleengine => common/queue/src/main/java/org/thingsboard/server/queue/common/consumer}/QueueEvent.java (92%) rename {application/src/main/java/org/thingsboard/server/service/queue/ruleengine => common/queue/src/main/java/org/thingsboard/server/queue/common/consumer}/TbQueueConsumerManagerTask.java (96%) rename {application/src/main/java/org/thingsboard/server/service/queue/ruleengine => common/queue/src/main/java/org/thingsboard/server/queue/common/consumer}/TbQueueConsumerTask.java (89%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java create mode 100644 common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java rename dao/src/main/java/org/thingsboard/server/dao/sql/query/{QueryContext.java => SqlQueryContext.java} (94%) create mode 100644 edqs/pom.xml create mode 100644 edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java create mode 100644 edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java create mode 100644 edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java create mode 100644 edqs/src/main/resources/edqs.yml create mode 100644 edqs/src/main/resources/logback.xml create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java create mode 100644 edqs/src/test/resources/edq-test.properties diff --git a/application/pom.xml b/application/pom.xml index d97f6d92e3..9bdfc7c754 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -124,6 +124,10 @@ org.thingsboard.common edge-api + + org.thingsboard.common + edqs + org.thingsboard dao diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 505244539b..f28d485b30 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -20,6 +20,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -27,6 +28,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -38,6 +40,7 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.query.EntityQueryService; @@ -55,6 +58,8 @@ public class EntityQueryController extends BaseController { @Autowired private EntityQueryService entityQueryService; + @Autowired + private EdqsService edqsService; private static final int MAX_PAGE_SIZE = 100; @@ -133,4 +138,10 @@ public class EntityQueryController extends BaseController { return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope); } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @PostMapping("/edqs/system/request") + public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) { + edqsService.processSystemRequest(request); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 9317c51a64..c7e95dbbc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -82,6 +82,11 @@ public class EdgeEventSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + try { if (!isValidSaveEntityEventForEdgeProcessing(event)) { return; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java new file mode 100644 index 0000000000..7582d036e2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -0,0 +1,340 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.ByteString; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.edqs.processor.EdqsConverter; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.environment.DistributedLock; +import org.thingsboard.server.queue.environment.DistributedLockService; +import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +public class DefaultEdqsService implements EdqsService { + + private final EdqsClientQueueFactory queueFactory; + private final EdqsConverter edqsConverter; + private final EdqsSyncService edqsSyncService; + private final DistributedLockService distributedLockService; + private final AttributesService attributesService; + private final EdqsPartitionService edqsPartitionService; + @Autowired @Lazy + private TbClusterService clusterService; + @Autowired @Lazy + private HashPartitionService hashPartitionService; + + @Value("${queue.edqs.api_enabled:false}") + private Boolean apiEnabled; + + private EdqsProducer eventsProducer; + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + private ExecutorService executor; + private DistributedLock syncLock; + + @PostConstruct + private void init() { + executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); + eventsProducer = EdqsProducer.builder() + .queue(EdqsQueue.EVENTS) + .partitionService(edqsPartitionService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) + .build(); + if (apiEnabled) { + apiEnabled = null; + } + + requestTemplate = queueFactory.createEdqsRequestTemplate(); + requestTemplate.init(); + syncLock = distributedLockService.getLock("edqs_sync"); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onStartUp() { + executor.submit(() -> { + try { + EdqsSyncState syncState = getSyncState(); + if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { + if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { + processSystemRequest(ToCoreEdqsRequest.builder() + .syncRequest(new EdqsSyncRequest()) + .build()); + } + } else { // only if topic/RocksDB is not empty and sync is finished + if (apiEnabled == null) { + log.info("EDQS is already synced, enabling API"); + apiEnabled = true; + } else { + log.info("EDQS is already synced"); + } + } + } catch (Throwable e) { + log.error("Failed to start EDQS service", e); + } + }); + } + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) { + log.info("Processing system request {}", request); + if (request.getSyncRequest() != null) { + saveSyncState(EdqsSyncStatus.REQUESTED); + } + broadcast(request.toInternalMsg()); + } + + @Override + public void processSystemMsg(ToCoreEdqsMsg msg) { + executor.submit(() -> { + log.info("Processing system msg {}", msg); + try { + if (msg.getApiEnabled() != null) { + apiEnabled = msg.getApiEnabled(); + } + + if (msg.getSyncRequest() != null) { + syncLock.lock(); + try { + EdqsSyncState syncState = getSyncState(); + if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) { + log.info("EDQS sync is already finished"); + return; + } + + saveSyncState(EdqsSyncStatus.STARTED); + edqsSyncService.sync(); + + saveSyncState(EdqsSyncStatus.FINISHED); + if (apiEnabled == null) { + broadcast(ToCoreEdqsMsg.builder() + .apiEnabled(Boolean.TRUE) + .build()); + } + } catch (Exception e) { + log.error("Failed to complete sync", e); + saveSyncState(EdqsSyncStatus.FAILED); + } finally { + syncLock.unlock(); + } + } + } catch (Throwable e) { + log.error("Failed to process msg {}", msg, e); + } + }); + } + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType); + return; + } + onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity)); + } + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.UPDATED, object); + } + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType); + return; + } + onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE)); + } + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.DELETED, object); + } + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + var requestMsg = newEdqsMsg(tenantId) + .setRequestMsg(EdqsRequestMsg.newBuilder() + .setValue(JacksonUtil.toString(request)) + .build()); + if (customerId != null && !customerId.isNullUid()) { + requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits()); + requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits()); + } + + Integer partition = edqsPartitionService.resolvePartition(tenantId); + ListenableFuture> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition); + return Futures.transform(resultFuture, msg -> { + TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg(); + return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class); + }, MoreExecutors.directExecutor()); + } + + @Override + public boolean isApiEnabled() { + return Boolean.TRUE.equals(apiEnabled); + } + + protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) { + executor.submit(() -> { + try { + String key = object.key(); + Long version = object.version(); + EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder() + .setKey(key) + .setObjectType(objectType.name()) + .setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object))) + .setEventType(eventType.name()); + if (version != null) { + eventMsg.setVersion(version); + } + eventsProducer.send(tenantId, objectType, key, newEdqsMsg(tenantId) + .setEventMsg(eventMsg) + .build()); + } catch (Throwable e) { + log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e); + } + }); + } + + private boolean isEdqsType(TenantId tenantId, ObjectType objectType) { + if (objectType == null) { + return false; + } + if (!tenantId.isSysTenantId()) { + return ObjectType.edqsTypes.contains(objectType); + } else { + return ObjectType.edqsSystemTypes.contains(objectType); + } + } + + private void broadcast(ToCoreEdqsMsg msg) { + clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder() + .setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder() + .setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg)))) + .build()); + } + + private static ToEdqsMsg.Builder newEdqsMsg(TenantId tenantId) { + return ToEdqsMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTs(System.currentTimeMillis()); + } + + @PreDestroy + private void preDestroy() { + executor.shutdown(); + eventsProducer.stop(); + requestTemplate.stop(); + } + + @SneakyThrows + private EdqsSyncState getSyncState() { + EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS) + .flatMap(KvEntry::getJsonValue) + .map(value -> JacksonUtil.fromString(value, EdqsSyncState.class)) + .orElse(null); + log.info("getSyncState = {}", state); + return state; + } + + @SneakyThrows + private void saveSyncState(EdqsSyncStatus status) { + EdqsSyncState state = new EdqsSyncState(status); + log.info("saveSyncState {}", state); + attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry( + new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)), + System.currentTimeMillis())).get(30, TimeUnit.SECONDS); + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + private static class EdqsSyncState { + private EdqsSyncStatus status; + } + + private enum EdqsSyncStatus { + REQUESTED, + STARTED, + FINISHED, + FAILED + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java new file mode 100644 index 0000000000..69a3f6108d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java @@ -0,0 +1,539 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.converter.Converter; +import org.thingsboard.server.common.data.converter.ConverterType; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.ConverterId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.IntegrationId; +import org.thingsboard.server.common.data.id.RoleId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.SchedulerEventId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.integration.Integration; +import org.thingsboard.server.common.data.integration.IntegrationType; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.role.Role; +import org.thingsboard.server.common.data.role.RoleType; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.scheduler.SchedulerEvent; +import org.thingsboard.server.common.data.widget.WidgetType; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.edqs.processor.EdqsConverter; + +import java.io.FileReader; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +import static org.thingsboard.common.util.JacksonUtil.toJsonNode; + + +@RequiredArgsConstructor +@Slf4j +//@Service +public class EdqsDataLoader { + + private final EdqsService edqsService; + private final EdqsConverter edqsConverter; + + public final static TenantId MAIN = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); + + private final String folder = "/home/viacheslav/Downloads/schwarz"; + + private ExecutorService executor = Executors.newFixedThreadPool(5, ThingsBoardThreadFactory.forName("edqs-publisher")); + +// @AfterStartUp(order = 100) + public void load() throws Exception { + loadCustomers(); + loadDeviceProfile(); + loadDevices(); + loadAssets(); + loadEdges(); + loadEntityViews(); + loadTenants(); + loadUsers(); + loadDashboards(); + loadRuleChains(); + loadWidgetType(); + loadWidgetBundle(); + loadConverters(); + loadIntegrations(); + loadSchedulerEvents(); + loadRoles(); + loadApiUsageStates(); + loadAssetProfile(); + loadEntityGroups(); + loadRelations(); + + loadAttributes(); + loadTs(); + } + + private void loadCustomers() throws Exception { + load("customer.csv", (values) -> { + Customer customer = new Customer(); + customer.setTitle(values.get("title")); + customer.setId(new CustomerId(UUID.fromString(values.get("id")))); + customer.setCreatedTime(Long.parseLong(values.get("created_time"))); + customer.setTenantId(tenantId(values.get("tenant_id"))); + var parentCustomerId = values.get("parent_customer_id"); + if (StringUtils.isNotEmpty(parentCustomerId)) { + customer.setParentCustomerId(new CustomerId(UUID.fromString(parentCustomerId))); + } + edqsService.onUpdate(customer.getTenantId(), customer.getId(), customer); + }); + } + + private void loadDevices() throws Exception { + load("device.csv", (values) -> { + Device device = new Device(); + device.setType(values.get("type")); + device.setName(values.get("name")); + device.setLabel(values.get("label")); + device.setId(new DeviceId(uuid(values.get("id")))); + device.setCreatedTime(parseLong(values.get("created_time"))); + device.setCustomerId(customerId(values.get("customer_id"))); + device.setTenantId(tenantId(values.get("tenant_id"))); + device.setDeviceProfileId(new DeviceProfileId(uuid(values.get("device_profile_id")))); + device.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(device.getTenantId(), device.getId(), device); + }); + } + + private void loadAssets() throws Exception { + load("asset.csv", (values) -> { + Asset asset = new Asset(); + asset.setType(values.get("type")); + asset.setName(values.get("name")); + asset.setLabel(values.get("label")); + asset.setId(new AssetId(uuid(values.get("id")))); + asset.setCreatedTime(parseLong(values.get("created_time"))); + asset.setCustomerId(customerId(values.get("customer_id"))); + asset.setTenantId(tenantId(values.get("tenant_id"))); + asset.setAssetProfileId(new AssetProfileId(uuid(values.get("asset_profile_id")))); + asset.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(asset.getTenantId(), asset.getId(), asset); + }); + } + + private void loadEdges() throws Exception { + load("edge.csv", (values) -> { + Edge edge = new Edge(); + edge.setId(new EdgeId(uuid(values.get("id")))); + edge.setCreatedTime(parseLong(values.get("created_time"))); + edge.setType(values.get("type")); + edge.setName(values.get("name")); + edge.setLabel(values.get("label")); + edge.setCustomerId(customerId(values.get("customer_id"))); + edge.setTenantId(tenantId(values.get("tenant_id"))); + edge.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(edge.getTenantId(), edge.getId(), edge); + }); + } + + private void loadEntityViews() throws Exception { + load("entity_view.csv", (values) -> { + EntityView entityView = new EntityView(); + entityView.setId(new EntityViewId(uuid(values.get("id")))); + entityView.setCreatedTime(parseLong(values.get("created_time"))); + entityView.setType(values.get("type")); + entityView.setName(values.get("name")); + entityView.setCustomerId(customerId(values.get("customer_id"))); + entityView.setTenantId(tenantId(values.get("tenant_id"))); + entityView.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(entityView.getTenantId(), entityView.getId(), entityView); + }); + } + + private void loadTenants() throws Exception { + load("tenant.csv", (values) -> { + Tenant tenant = new Tenant(); + tenant.setId(new TenantId(uuid(values.get("id")))); + tenant.setCreatedTime(parseLong(values.get("created_time"))); + tenant.setEmail(values.get("email")); + tenant.setTitle(values.get("title")); + tenant.setCountry(values.get("country")); + tenant.setState(values.get("state")); + tenant.setCity(values.get("city")); + tenant.setAddress(values.get("address")); + tenant.setAddress2(values.get("address2")); + tenant.setZip(values.get("zip")); + tenant.setPhone(values.get("phone")); + tenant.setRegion(values.get("region")); + tenant.setTenantProfileId(new TenantProfileId(uuid(values.get("tenant_profile_id")))); + tenant.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + edqsService.onUpdate(MAIN, tenant.getId(), tenant); + }); + } + + private void loadUsers() throws Exception { + load("user.csv", (values) -> { + User user = new User(); + user.setId(new UserId(uuid(values.get("id")))); + user.setCreatedTime(parseLong(values.get("created_time"))); + user.setTenantId(tenantId(values.get("tenant_id"))); + user.setFirstName(values.get("first_name")); + user.setLastName(values.get("last_name")); + user.setEmail(values.get("email")); + user.setPhone(values.get("phone")); + user.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(user.getTenantId(), user.getId(), user); + }); + } + + private void loadDashboards() throws Exception { + load("dashboard.csv", (values) -> { + Dashboard dashboard = new Dashboard(); + dashboard.setId(new DashboardId(uuid(values.get("id")))); + dashboard.setCreatedTime(parseLong(values.get("created_time"))); + dashboard.setTenantId(tenantId(values.get("tenant_id"))); + dashboard.setTitle(values.get("title")); + + edqsService.onUpdate(dashboard.getTenantId(), dashboard.getId(), dashboard); + }); + } + + private void loadEntityGroups() throws Exception { + load("entity_group.csv", (values) -> { + EntityGroup entityGroup = new EntityGroup(); + entityGroup.setId(new EntityGroupId(uuid(values.get("id")))); + entityGroup.setCreatedTime(parseLong(values.get("created_time"))); + entityGroup.setName(values.get("name")); + entityGroup.setOwnerId(entityId(values.get("owner_type"), values.get("owner_id"))); + entityGroup.setType(EntityType.valueOf(values.get("type"))); + edqsService.onUpdate(MAIN, entityGroup.getId(), entityGroup); + }); + } + + private void loadRelations() throws Exception { + load("relation.csv", (values) -> { + EntityRelation entityRelation = new EntityRelation(); + entityRelation.setFrom(entityId(values.get("from_type"), values.get("from_id"))); + entityRelation.setTo(entityId(values.get("to_type"), values.get("to_id"))); + entityRelation.setTypeGroup(RelationTypeGroup.valueOf(values.get("relation_type_group"))); + entityRelation.setType(values.get("relation_type")); + edqsService.onUpdate(MAIN, ObjectType.RELATION, entityRelation); + }); + } + + private void loadRuleChains() throws Exception { + load("rule_chain.csv", (values) -> { + RuleChain ruleChain = new RuleChain(); + ruleChain.setId(new RuleChainId(uuid(values.get("id")))); + ruleChain.setCreatedTime(parseLong(values.get("created_time"))); + ruleChain.setName(values.get("name")); + ruleChain.setTenantId(tenantId(values.get("tenant_id"))); + ruleChain.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(ruleChain.getTenantId(), ruleChain.getId(), ruleChain); + }); + } + + private void loadWidgetType() throws Exception { + load("widget_type.csv", (values) -> { + WidgetType widgetType = new WidgetType(); + widgetType.setId(new WidgetTypeId(uuid(values.get("id")))); + widgetType.setCreatedTime(parseLong(values.get("created_time"))); + widgetType.setName(values.get("name")); + widgetType.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(widgetType.getTenantId(), widgetType.getId(), widgetType); + }); + } + + private void loadWidgetBundle() throws Exception { + load("widgets_bundle.csv", (values) -> { + WidgetsBundle widgetsBundle = new WidgetsBundle(); + widgetsBundle.setId(new WidgetsBundleId(uuid(values.get("id")))); + widgetsBundle.setCreatedTime(parseLong(values.get("created_time"))); + widgetsBundle.setTitle(values.get("title")); + widgetsBundle.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle); + }); + } + + private void loadConverters() throws Exception { + load("converter.csv", (values) -> { + Converter converter = new Converter(); + converter.setId(new ConverterId(uuid(values.get("id")))); + converter.setCreatedTime(parseLong(values.get("created_time"))); + converter.setName(values.get("name")); + converter.setType(ConverterType.valueOf(values.get("type"))); + converter.setTenantId(tenantId(values.get("tenant_id"))); + converter.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); + converter.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(converter.getTenantId(), converter.getId(), converter); + }); + } + + private void loadIntegrations() throws Exception { + load("integration.csv", (values) -> { + Integration integration = new Integration(); + integration.setId(new IntegrationId(uuid(values.get("id")))); + integration.setCreatedTime(parseLong(values.get("created_time"))); + integration.setName(values.get("name")); + integration.setType(IntegrationType.valueOf(values.get("type"))); + integration.setTenantId(tenantId(values.get("tenant_id"))); + integration.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); + integration.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(integration.getTenantId(), integration.getId(), integration); + }); + } + + private void loadSchedulerEvents() throws Exception { + load("scheduler_event.csv", (values) -> { + SchedulerEvent schedulerEvent = new SchedulerEvent(); + schedulerEvent.setId(new SchedulerEventId(uuid(values.get("id")))); + schedulerEvent.setCreatedTime(parseLong(values.get("created_time"))); + schedulerEvent.setName(values.get("name")); + schedulerEvent.setType(values.get("type")); + schedulerEvent.setTenantId(tenantId(values.get("tenant_id"))); + schedulerEvent.setConfiguration(toJsonNode(values.get("configuration"))); + schedulerEvent.setSchedule(toJsonNode(values.get("schedule"))); + schedulerEvent.setOriginatorId(entityId(values.get("originator_type"), values.get("originator_id"))); + schedulerEvent.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(schedulerEvent.getTenantId(), schedulerEvent.getId(), schedulerEvent); + }); + } + + private void loadRoles() throws Exception { + load("role.csv", (values) -> { + Role role = new Role(); + role.setId(new RoleId(uuid(values.get("id")))); + role.setCreatedTime(parseLong(values.get("created_time"))); + role.setName(values.get("name")); + role.setType(RoleType.valueOf(values.get("type"))); + role.setTenantId(tenantId(values.get("tenant_id"))); + role.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(role.getTenantId(), role.getId(), role); + }); + } + + private void loadApiUsageStates() throws Exception { + load("api_usage_state.csv", (values) -> { + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setId(new ApiUsageStateId(uuid(values.get("id")))); + apiUsageState.setCreatedTime(parseLong(values.get("created_time"))); + apiUsageState.setEntityId(entityId(values.get("entity_type"), values.get("entity_id"))); + apiUsageState.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState); + }); + } + + private void loadDeviceProfile() throws Exception { + load("device_profile.csv", (values) -> { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setId(new DeviceProfileId(uuid(values.get("id")))); + deviceProfile.setCreatedTime(parseLong(values.get("created_time"))); + deviceProfile.setName(values.get("name")); + deviceProfile.setType(DeviceProfileType.valueOf(values.get("type"))); + deviceProfile.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile); + }); + } + + private void loadAssetProfile() throws Exception { + load("asset_profile.csv", (values) -> { + AssetProfile assetProfile = new AssetProfile(); + assetProfile.setId(new AssetProfileId(uuid(values.get("id")))); + assetProfile.setCreatedTime(parseLong(values.get("created_time"))); + assetProfile.setName(values.get("name")); + assetProfile.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(assetProfile.getTenantId(), assetProfile.getId(), assetProfile); + }); + } + + private void loadAttributes() throws Exception { + load("attribute.csv", (values) -> { + EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("entity_type"), values.get("entity_id")); + long ts = parseLong(values.get("last_update_ts")); + AttributeScope scope = AttributeScope.valueOf(values.get("attribute_type")); + String key = values.get("attribute_key"); + KvEntry kvEntry; + if (StringUtils.isNotEmpty(values.get("bool_v"))) { + kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); + } else if (StringUtils.isNotEmpty(values.get("str_v"))) { + kvEntry = new StringDataEntry(key, values.get("str_v")); + } else if (StringUtils.isNotEmpty(values.get("long_v"))) { + kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); + } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { + kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); + } else if (StringUtils.isNotEmpty(values.get("json_v"))) { + kvEntry = new JsonDataEntry(key, values.get("json_v")); + } else { + kvEntry = new StringDataEntry(key, ""); + } + AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(ts, kvEntry); + AttributeKv attributeKv = new AttributeKv(entityId, scope, attributeKvEntry, 0); + edqsService.onUpdate(MAIN, ObjectType.ATTRIBUTE_KV, attributeKv); + }); + } + + private void loadTs() throws Exception { + load("ts_kv.csv", (values) -> { + var entityTypeStr = values.get("find_entity_type"); + if (StringUtils.isEmpty(entityTypeStr)) { + return; + } + EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("find_entity_type"), values.get("entity_id")); + long ts = parseLong(values.get("ts")); + String key = values.get("key"); + KvEntry kvEntry; + if (StringUtils.isNotEmpty(values.get("bool_v"))) { + kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); + } else if (StringUtils.isNotEmpty(values.get("str_v"))) { + kvEntry = new StringDataEntry(key, values.get("str_v")); + } else if (StringUtils.isNotEmpty(values.get("long_v"))) { + kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); + } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { + kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); + } else if (StringUtils.isNotEmpty(values.get("json_v"))) { + kvEntry = new JsonDataEntry(key, values.get("json_v")); + } else { + kvEntry = new StringDataEntry(key, ""); + } + BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(ts, kvEntry); + edqsService.onUpdate(MAIN, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, 0L)); + }); + } + + private void load(String file, Consumer> function) throws Exception { + Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("loader-" + file)).submit(() -> { + try { + long ts = System.currentTimeMillis(); + CsvSchema schema = CsvSchema.emptySchema().withHeader().withColumnSeparator('|'); + CsvMapper mapper = new CsvMapper(); + MappingIterator> it = mapper + .readerFor(Map.class) + .with(schema) + .readValues(new FileReader(folder + "/" + file)); + + int success = 0; + int failure = 0; + while (it.hasNextValue()) { + Map row = it.nextValue(); + try { + function.accept(row); + success++; + if (success % 1000 == 0) { + log.info("Loaded [{}] from [{}]", success, file); + } + } catch (Exception e) { + log.error("Failed to parse str: [{}]", row, e); + failure++; + } + } + log.info("Loaded [{}] from [{}] in {}ms. Failures {}", success, file, (System.currentTimeMillis() - ts), failure); + } catch (Throwable t) { + log.error("Failed to load data from [{}]", file, t); + } + }); + } + + private static TenantId tenantId(String id) { + return TenantId.fromUUID(UUID.fromString(id)); + } + + private static CustomerId customerId(String id) { + var c = new CustomerId(UUID.fromString(id)); + return c.isNullUid() ? null : c; + } + + private static EntityId entityId(String type, String id) { + return EntityIdFactory.getByTypeAndId(type, id); + } + + private static UUID uuid(String id) { + return UUID.fromString(id); + } + + private static long parseLong(String time) { + return Long.parseLong(time); + } + + private static boolean parseBoolean(String value) { + return Boolean.parseBoolean(value); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java new file mode 100644 index 0000000000..c4ce0c7a7e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +public class EdqsListener { + + private final EdqsService edqsService; + + @TransactionalEventListener(fallbackExecution = true) + public void onUpdate(SaveEntityEvent event) { + if (event.getEntityId() == null || event.getEntity() == null) { + return; + } + edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void onDelete(DeleteEntityEvent event) { + if (event.getEntityId() == null) { + return; + } + edqsService.onDelete(event.getTenantId(), event.getEntityId()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java new file mode 100644 index 0000000000..8e12da56e0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -0,0 +1,275 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import org.thingsboard.server.dao.group.EntityGroupDao; +import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.relation.RelationDao; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; + +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.ObjectType.API_USAGE_STATE; +import static org.thingsboard.server.common.data.ObjectType.ASSET; +import static org.thingsboard.server.common.data.ObjectType.ASSET_PROFILE; +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.BLOB_ENTITY; +import static org.thingsboard.server.common.data.ObjectType.CONVERTER; +import static org.thingsboard.server.common.data.ObjectType.CUSTOMER; +import static org.thingsboard.server.common.data.ObjectType.DASHBOARD; +import static org.thingsboard.server.common.data.ObjectType.DEVICE; +import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE; +import static org.thingsboard.server.common.data.ObjectType.EDGE; +import static org.thingsboard.server.common.data.ObjectType.ENTITY_GROUP; +import static org.thingsboard.server.common.data.ObjectType.ENTITY_VIEW; +import static org.thingsboard.server.common.data.ObjectType.INTEGRATION; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.QUEUE_STATS; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.ROLE; +import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN; +import static org.thingsboard.server.common.data.ObjectType.SCHEDULER_EVENT; +import static org.thingsboard.server.common.data.ObjectType.TENANT; +import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE; +import static org.thingsboard.server.common.data.ObjectType.USER; +import static org.thingsboard.server.common.data.ObjectType.WIDGETS_BUNDLE; +import static org.thingsboard.server.common.data.ObjectType.WIDGET_TYPE; + +@Slf4j +public abstract class EdqsSyncService { + + @Autowired + private EntityDaoRegistry entityDaoRegistry; + @Autowired + private TenantDao tenantDao; + @Autowired + private AttributesDao attributesDao; + @Autowired + private KeyDictionaryDao keyDictionaryDao; + @Autowired + private RelationDao relationDao; + @Autowired + private EntityGroupDao entityGroupDao; + @Autowired + private TimeseriesLatestDao timeseriesLatestDao; + @Autowired + @Lazy + private DefaultEdqsService edqsService; + + private final ConcurrentHashMap entityInfoMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap keys = new ConcurrentHashMap<>(); + + private final Map counters = new ConcurrentHashMap<>(); + + public static final Set edqsTenantTypes = EnumSet.of( + TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, + BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS + ); + + public abstract boolean isSyncNeeded(); + + public void sync() { + log.info("Synchronizing data to EDQS"); + long startTs = System.currentTimeMillis(); + counters.clear(); + + syncTenants(); + syncTenantEntities(); + syncEntityGroups(); + syncRelations(); + loadKeyDictionary(); + syncAttributes(); + syncLatestTimeseries(); + + counters.clear(); + log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); + } + + private void process(TenantId tenantId, ObjectType type, EdqsObject object) { + AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger()); + if (counter.incrementAndGet() % 10000 == 0) { + log.info("Processed {} {} objects", counter.get(), type); + } + edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); + } + + private void syncTenants() { + log.info("Synchronizing tenants to EDQS"); + long ts = System.currentTimeMillis(); + var tenants = new PageDataIterable<>(tenantDao::findAllFields, 10000); + for (EntityFields entityFields : tenants) { + TenantId tenantId = TenantId.fromUUID(entityFields.getId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(EntityType.TENANT, tenantId)); + process(tenantId, TENANT, new Entity(EntityType.TENANT, entityFields)); + } + process(TenantId.SYS_TENANT_ID, TENANT, new Entity(EntityType.TENANT, new TenantFields(TenantId.SYS_TENANT_ID.getId(), Long.MAX_VALUE))); + log.info("Finished synchronizing tenants to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncTenantEntities() { + for (ObjectType type : edqsTenantTypes) { + log.info("Synchronizing tenant {} entities to EDQS", type); + long ts = System.currentTimeMillis(); + EntityType entityType = type.toEntityType(); + Dao dao = entityDaoRegistry.getDao(entityType); + var entities = new PageDataIterable<>(dao::findAllFields, 10000); + for (EntityFields entityFields : entities) { + TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); + process(tenantId, type, new Entity(type.toEntityType(), entityFields)); + } + log.info("Finished synchronizing tenant {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); + } + } + + private void syncEntityGroups() { + log.info("Synchronizing entity groups to EDQS"); + long ts = System.currentTimeMillis(); + var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 10000); + for (EntityFields groupFields : entityGroups) { + EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId()); + if (entityIdInfo != null) { + entityInfoMap.put(groupFields.getId(), new EntityIdInfo(EntityType.ENTITY_GROUP, entityIdInfo.tenantId())); + process(entityIdInfo.tenantId(), ENTITY_GROUP, new Entity(EntityType.ENTITY_GROUP, groupFields)); + } else { + log.info("Entity group owner not found: " + groupFields.getOwnerId()); + } + } + log.info("Finished synchronizing entity groups to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncRelations() { + log.info("Synchronizing relations to EDQS"); + long ts = System.currentTimeMillis(); + var relations = new PageDataIterable<>(relationDao::findAll, 10000); + for (EntityRelation relation : relations) { + if (relation.getTypeGroup() == RelationTypeGroup.COMMON || relation.getTypeGroup() == RelationTypeGroup.FROM_ENTITY_GROUP) { + EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFrom().getId()); + if (entityIdInfo != null) { + process(entityIdInfo.tenantId(), RELATION, relation); + } else { + log.info("Relation from entity not found: " + relation.getFrom()); + } + } + } + log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void loadKeyDictionary() { + log.info("Loading key dictionary"); + long ts = System.currentTimeMillis(); + var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000); + for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) { + keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey()); + } + log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncAttributes() { + log.info("Synchronizing attributes to EDQS"); + long ts = System.currentTimeMillis(); + var attributes = new PageDataIterable<>(attributesDao::findAll, 10000); + for (AttributeKvEntity attribute : attributes) { + attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); + UUID entityId = attribute.getId().getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); + if (entityIdInfo == null) { + log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId); + continue; + } + AttributeKv attributeKv = new AttributeKv( + EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId), + AttributeScope.valueOf(attribute.getId().getAttributeType()), + attribute.toData(), + attribute.getVersion()); + process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); + } + log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncLatestTimeseries() { + log.info("Synchronizing latest timeseries to EDQS"); + long ts = System.currentTimeMillis(); + var tsKvLatestEntities = new PageDataIterable<>(pageLink -> timeseriesLatestDao.findAllLatest(pageLink), 10000); + for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { + try { + String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); + if (strKey == null) { + log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey()); + continue; + } + tsKvLatestEntity.setStrKey(strKey); + UUID entityUuid = tsKvLatestEntity.getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid); + if (entityIdInfo != null) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid); + LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion()); + process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv); + } + } catch (Exception e) { + log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); + } + } + log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private String getStrKeyOrFetchFromDb(int key) { + String strKey = keys.get(key); + if (strKey != null) { + return strKey; + } else { + strKey = keyDictionaryDao.getKey(key); + keys.putIfAbsent(key, strKey); + } + return strKey; + } + + public record EntityIdInfo(EntityType entityType, TenantId tenantId) {} + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java new file mode 100644 index 0000000000..d51d87ed01 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; + +import java.util.Collections; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") +public class KafkaEdqsSyncService extends EdqsSyncService { + + private final TbKafkaSettings kafkaSettings; + private TbKafkaAdmin kafkaAdmin; + + @PostConstruct + private void init() { + kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); + } + + @Override + public boolean isSyncNeeded() { + return kafkaAdmin.isTopicEmpty(EdqsQueue.STATE.getTopic()); + } + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java new file mode 100644 index 0000000000..924bf94830 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") +public class LocalEdqsSyncService extends EdqsSyncService { + + private final EdqsRocksDb db; + + @Override + public boolean isSyncNeeded() { + return db.isNew(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 80368c085f..3cc999df29 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -72,6 +72,11 @@ public class EntityStateSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); if (entityId == null) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3aaa36b068..1b4bfe7c71 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -35,6 +34,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.JavaSerDesUtil; import org.thingsboard.server.common.data.alarm.AlarmInfo; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -78,6 +79,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -89,7 +91,6 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.resource.TbImageService; @@ -147,9 +148,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; private QueueConsumerManager> firmwareStatesConsumer; @@ -175,7 +177,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) - .config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -251,7 +255,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, CoreQueueConfig config) throws Exception { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig config) throws Exception { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); @@ -396,6 +400,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, EdgeQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, StatsFactory statsFactory, EdgeContextComponent edgeCtx) { @@ -102,9 +102,9 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService, EdgeQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME)) - .config(EdgeQueueConfig.of(consumerPerPartition, pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -130,7 +130,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, EdgeQueueConfig edgeQueueConfig) throws InterruptedException { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig edgeQueueConfig) throws InterruptedException { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); @@ -323,10 +323,4 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService { + event.getNewPartitions().forEach((queueKey, partitions) -> { if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { var consumer = getConsumer(queueKey).orElseGet(() -> { Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c2823d3c00..3636eb05af 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -33,11 +33,14 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.QueueEvent; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; -import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index c58cb4fd5a..d3d0421297 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -251,7 +251,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService= deviceState.getLastActivityTime()) { deviceState.setLastInactivityAlarmTime(0L); - save(deviceId, INACTIVITY_ALARM_TIME, 0L); + save(state.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0L); } } } @@ -583,7 +583,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService>> customExporters; + private Map relatedEntitiesExporters; + + private static final Set RELATED = EnumSet.of(EVENT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV); + + @PostConstruct + private void init() { + relatedEntitiesExporters = Map.of( + RELATION, this::exportRelations, + EVENT, this::exportEvents, // todo: query by tenant + ATTRIBUTE_KV, this::exportAttributes, + LATEST_TS_KV, this::exportLatestTelemetry + ); + customExporters = Map.of( + AUDIT_LOG, this::exportAuditLogs + ); + } + + public void exportTenant(TenantId tenantId, ExportConfig config, BiConsumer processor) { + log.info("[{}] Exporting tenant", tenantId); + Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); + if (tenant == null) { + throw new IllegalArgumentException("Tenant with id " + tenantId + " not found"); + } + + Set objectTypes = config.getIncludedObjectTypes(); + if (objectTypes.contains(TENANT)) { + exportEntity(tenantId, TENANT, tenant, config, processor); + } + + for (ObjectType type : objectTypes) { + if (RELATED.contains(type) || type == TENANT) { + continue; + } + log.debug("[{}] Exporting {} entities", tenantId, type); + if (!customExporters.containsKey(type)) { + TenantEntityDao dao = entityDaoRegistry.getTenantEntityDao(type); + var entities = new PageDataIterable<>(pageLink -> dao.findAllByTenantId(tenantId, pageLink), 100); + for (Object entity : entities) { + exportEntity(tenantId, type, entity, config, processor); + } + } else { + customExporters.get(type).accept(tenantId, processor); + } + } + } + + private void exportEntity(TenantId tenantId, ObjectType type, Object entity, ExportConfig config, BiConsumer processor) { + processor.accept(type, entity); + if (entity instanceof HasId hasId && hasId.getId() instanceof EntityId entityId) { + relatedEntitiesExporters.forEach((relatedEntityType, exporter) -> { + if (config.getIncludedObjectTypes().contains(relatedEntityType)) { + exporter.export(tenantId, entityId, processor); + } + }); + } + } + + private Map getPartitions(String table) { + List partitionsStartTime = partitioningRepository.fetchPartitions(table).stream().sorted().toList(); + if (partitionsStartTime.isEmpty()) { + return Collections.emptyMap(); + } + + Map partitions = new HashMap<>(); + for (int i = 0; i < partitionsStartTime.size(); i++) { + Long startTime = partitionsStartTime.get(i); + Long endTime; + if (partitionsStartTime.size() - 1 == i) { + endTime = System.currentTimeMillis(); + } else { + endTime = partitionsStartTime.get(i + 1) - 1; + } + partitions.put(startTime, endTime); + } + return partitions; + } + + private void exportAuditLogs(TenantId tenantId, BiConsumer processor) { + Map partitions = getPartitions(ModelConstants.AUDIT_LOG_TABLE_NAME); + partitions.forEach((startTime, endTime) -> { + PageDataIterable auditLogs = new PageDataIterable<>(pageLink -> { + return auditLogDao.findAuditLogsByTenantId(tenantId.getId(), null, new TimePageLink(pageLink, startTime, endTime)); + }, 512); + for (AuditLog auditLog : auditLogs) { + processor.accept(AUDIT_LOG, auditLog); + } + }); + } + + private void exportAttributes(TenantId tenantId, EntityId entityId, BiConsumer processor) { + for (AttributeScope attributeScope : AttributeScope.values()) { + List attributes = attributesDao.findAll(tenantId, entityId, attributeScope); + for (AttributeKvEntry entry : attributes) { + AttributeKv attributeKv = new AttributeKv(entityId, attributeScope, entry, entry.getVersion()); + processor.accept(ATTRIBUTE_KV, attributeKv); + } + } + } + + private void exportRelations(TenantId tenantId, EntityId entityId, BiConsumer processor) { + List relations = relationDao.findAllByFrom(tenantId, entityId); + for (EntityRelation relation : relations) { + processor.accept(RELATION, relation); + } + } + + @SneakyThrows + private void exportLatestTelemetry(TenantId tenantId, EntityId entityId, BiConsumer processor) { + List latestTelemetry = timeseriesLatestDao.findAllLatest(tenantId, entityId).get(30, TimeUnit.SECONDS); + for (TsKvEntry tsKvEntry : latestTelemetry) { + LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvEntry, tsKvEntry.getVersion()); + processor.accept(LATEST_TS_KV, latestTsKv); + } + } + + private void exportEvents(TenantId tenantId, EntityId entityId, BiConsumer processor) { + for (EventType eventType : EventType.values()) { + Map partitions = getPartitions(eventType.getTable()); + partitions.forEach((startTime, endTime) -> { + PageDataIterable events = new PageDataIterable<>(pageLink -> { + return eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, new TimePageLink(pageLink, startTime, endTime)); + }, 512); + for (Event event : events) { + processor.accept(EVENT, event); + } + }); + } + } + + private interface Exporter { + + void export(TenantId tenantId, EntityId entityId, BiConsumer processor); + + } + + @Data + public static class ExportConfig { + + private Set includedObjectTypes; + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index b38fbf4fec..c0360ede8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -93,7 +93,8 @@ public class TbCoreTransportApiService { @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(transportApiService); + transportApiTemplate.subscribe(); + transportApiTemplate.launch(transportApiService); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java index 34fd4a426f..2bd39b8c09 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java @@ -229,6 +229,7 @@ public class DefaultWebSocketService implements WebSocketService { } catch (TbRateLimitsException e) { log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } catch (Exception e) { + sendError(sessionRef, cmd.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR, e.getMessage()); log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f29cc55197..9b182ce3a7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1615,6 +1615,12 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1688,6 +1694,23 @@ queue: enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" + edqs: + sync_enabled: "${TB_EDQS_SYNC_ENABLED:true}" # FIXME: disable by default before release + api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release + mode: "${TB_EDQS_MODE:local}" # local or remote + local: + rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:/tmp/edqs-backup}" + partitions: "${TB_EDQS_PARTITIONS:12}" + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + stats: + # Enable/disable statistics for EDQS service + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" vc: # Default topic name diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java new file mode 100644 index 0000000000..daa1c92c1c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.type=kafka", // uncomment to use Kafka + "queue.kafka.bootstrap.servers=10.7.1.254:9092", + "queue.edqs.sync_enabled=true", + "queue.edqs.api_enabled=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { + + @Autowired + private EdqsService edqsService; + + @MockBean + private EdqsRocksDb edqsRocksDb; + + @Before + public void before() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); + } + + @Override + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == expectedResult); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 5d04b16dfa..ad554f11ba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.awaitility.Awaitility; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -49,6 +51,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; @@ -73,6 +76,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @@ -130,36 +134,25 @@ public class EntityQueryControllerTest extends AbstractControllerTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); - countQuery = new EntityCountQuery(entityListFilter); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); EntityTypeFilter filter2 = new EntityTypeFilter(); filter2.setEntityType(EntityType.DEVICE); - - EntityCountQuery countQuery2 = new EntityCountQuery(filter2); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class); - Assert.assertEquals(97, count2.longValue()); + countQuery = new EntityCountQuery(filter2); + countByQueryAndCheck(countQuery, 97); } @Test @@ -169,14 +162,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); allDeviceFilter.setEntityType(EntityType.DEVICE); EntityCountQuery query = new EntityCountQuery(allDeviceFilter); - Long initialCount = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); + Long initialCount = countByQuery(query); loginTenantAdmin(); List devices = new ArrayList<>(); + String devicePrefix = "Device" + RandomStringUtils.random(5); for (int i = 0; i < 97; i++) { Device device = new Device(); - device.setName("Device" + i); + device.setName(devicePrefix + i); device.setType("default"); device.setLabel("testLabel" + (int) (Math.random() * 1000)); devices.add(doPost("/api/device", device, Device.class)); @@ -189,31 +183,23 @@ public class EntityQueryControllerTest extends AbstractControllerTest { loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, initialCount + 97); filter.setDeviceType("unknown"); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); + countByQueryAndCheck(countQuery, 0); filter.setDeviceType("default"); - filter.setDeviceNameFilter("Device1"); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + filter.setDeviceNameFilter(devicePrefix + "1"); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); + countByQueryAndCheck(countQuery, 97); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); - Assert.assertEquals(initialCount + 97, count2.longValue()); + countByQueryAndCheck(query, initialCount + 97); } @Test @@ -371,11 +357,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -383,8 +365,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -406,8 +387,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -423,9 +403,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null); - PageData data2 = - doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference>() { - }); + PageData data2 = findByQuery(query2); Assert.assertEquals(97, data2.getTotalElements()); Assert.assertEquals(10, data2.getTotalPages()); @@ -473,20 +451,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(10, data.getTotalElements()); + findByQueryAndCheck(query, 10); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); } private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) { @@ -531,14 +504,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -551,6 +522,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(45)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); @@ -559,13 +531,11 @@ public class EntityQueryControllerTest extends AbstractControllerTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -604,6 +574,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); DynamicValue dynamicValue = @@ -627,16 +598,16 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - Awaitility.await() + await() .alias("data by query") .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == numOfDevices; }); - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(numOfDevices, loadedEntities.size()); @@ -694,11 +665,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -712,9 +679,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { }); EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); } @Test @@ -742,28 +707,28 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); // all devices with ownerName = TEST TENANT - EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); - checkEntitiesCount(query, numOfDevices); + EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); + countByQueryAndCheck(query, numOfDevices); // all devices with ownerName = TEST TENANT - EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); - checkEntitiesCount(activeAlarmTimeToLongQuery, 0); + EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); + countByQueryAndCheck(activeAlarmTimeToLongQuery, 0); // all devices with wrong ownerName EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter)); - checkEntitiesCount(wrongTenantNameQuery, 0); + countByQueryAndCheck(wrongTenantNameQuery, 0); // all devices with owner type = TENANT EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter)); - checkEntitiesCount(tenantEntitiesQuery, numOfDevices); + countByQueryAndCheck(tenantEntitiesQuery, numOfDevices); // all devices with owner type = CUSTOMER EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter)); - checkEntitiesCount(customerEntitiesQuery, 0); + countByQueryAndCheck(customerEntitiesQuery, 0); } @Test @@ -790,7 +755,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); EntityDataSortOrder sortOrder = new EntityDataSortOrder( @@ -874,18 +839,18 @@ public class EntityQueryControllerTest extends AbstractControllerTest { } private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { - Awaitility.await() + await() .alias("data by query") .atMost(30, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == expectedNumOfDevices; }); if (expectedNumOfDevices == 0) { return; } - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(expectedNumOfDevices, loadedEntities.size()); @@ -898,25 +863,36 @@ public class EntityQueryControllerTest extends AbstractControllerTest { String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue(); Assert.assertEquals("Device" + i, name); - Assert.assertEquals( expectedOwnerName, ownerName); - Assert.assertEquals( expectedOwnerType, ownerType); + Assert.assertEquals(expectedOwnerName, ownerName); + Assert.assertEquals(expectedOwnerType, ownerType); Assert.assertEquals("1" + i, alarmActiveTime); } } - private void checkEntitiesCount(EntityCountQuery query, int expectedNumOfDevices) { - Awaitility.await() - .alias("count by query") - .atMost(30, TimeUnit.SECONDS) - .until(() -> { - var count = doPost("/api/entitiesQuery/count", query, Integer.class); - return count == expectedNumOfDevices; - }); - } + protected PageData findByQuery(EntityDataQuery query) throws Exception { + return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); + } + + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception { + PageData result = findByQuery(query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected Long countByQuery(EntityCountQuery countQuery) throws Exception { + return doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + } + + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) throws Exception { + Long result = countByQuery(query); + assertThat(result).isEqualTo(expectedResult); + return result; + } private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) { KeyFilter tenantOwnerNameFilter = new KeyFilter(); tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName)); + tenantOwnerNameFilter.setValueType(EntityKeyValueType.STRING); StringFilterPredicate ownerNamePredicate = new StringFilterPredicate(); ownerNamePredicate.setValue(FilterPredicateValue.fromString(value)); ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL); @@ -927,6 +903,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) { KeyFilter numericFilter = new KeyFilter(); numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute)); + numericFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(value)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 52cbaa4add..0a4009e8b2 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -153,7 +153,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()); + ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()).get(0); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -308,7 +308,7 @@ public class HashPartitionServiceTest { partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID); - return event.getPartitionsMap().get(queueKey).size() == systemQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == systemQueue.getPartitions(); }); Mockito.reset(applicationEventPublisher); @@ -336,14 +336,14 @@ public class HashPartitionServiceTest { // expecting event about no partitions for isolated queue key verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); partitionService_dedicated.updateQueues(List.of(queueUpdateMsg)); partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).size() == isolatedQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == isolatedQueue.getPartitions(); }); @@ -361,7 +361,7 @@ public class HashPartitionServiceTest { partitionService_dedicated.removeQueues(List.of(queueDeleteMsg)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); } @@ -381,12 +381,12 @@ public class HashPartitionServiceTest { Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> { List queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10)) .map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId)) - .limit(100).collect(Collectors.toList()); + .limit(100).toList(); for (int partition = 0; partition < 10; partition++) { - ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()); + ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()).get(0); for (QueueKey queueKey : queues) { - ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()); + ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()).get(0); assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId()) .isEqualTo(expectedAssignedRuleEngine); } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java new file mode 100644 index 0000000000..eea7e0eb7f --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy; + +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.edqs.sync_enabled=true", + "queue.edqs.api_enabled=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityServiceTest extends EntityServiceTest { + + @Autowired + private EdqsService edqsService; + + @MockBean + private EdqsRocksDb edqsRocksDb; + + @Before + public void beforeEach() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); + } + + @Override + protected PageData findByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityDataQuery query, long expectedResultSize) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, permissions, query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), mergedUserPermissionsPE, countQuery, expectedResult); + } + + @Override + protected long countByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityCountQuery query, int expectedResult) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, permissions, query), + result -> result == expectedResult); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java similarity index 87% rename from dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java rename to application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index a26f345fb7..dc29cec3cc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -13,22 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.service; +package org.thingsboard.server.service.entitiy; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; @@ -37,6 +40,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -46,6 +50,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; @@ -63,6 +68,7 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; @@ -75,17 +81,22 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardDao; +import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewDao; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; @@ -111,7 +122,7 @@ import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIEL @Slf4j @DaoSqlTest -public class EntityServiceTest extends AbstractServiceTest { +public class EntityServiceTest extends AbstractControllerTest { static final int ENTITY_COUNT = 5; public static final String TEST_CUSTOMER_NAME = "Test"; @@ -119,6 +130,12 @@ public class EntityServiceTest extends AbstractServiceTest { @Autowired AssetService assetService; @Autowired + AssetProfileService assetProfileService; + @Autowired + DashboardService dashboardService; + @Autowired + EntityViewService entityViewService; + @Autowired UserService userService; @Autowired AttributesService attributesService; @@ -157,7 +174,7 @@ public class EntityServiceTest extends AbstractServiceTest { } @Test - public void testCountEntitiesByQuery() throws InterruptedException { + public void testCountEntitiesByQuery() { List devices = new ArrayList<>(); for (int i = 0; i < 97; i++) { Device device = new Device(); @@ -173,30 +190,24 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); deviceService.deleteDevicesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @@ -211,19 +222,15 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(31, count); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) + countByQueryAndCheck(countQuery, 31); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter.setRootEntity(devices.get(0).getId()); filter.setDirection(EntitySearchDirection.TO); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Manages", Collections.singletonList(EntityType.TENANT)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(1, count); + countByQueryAndCheck(countQuery, 1); DeviceSearchQueryFilter filter2 = new DeviceSearchQueryFilter(); filter2.setRootEntity(tenantId); @@ -231,18 +238,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter2.setRelationType("Contains"); countQuery = new EntityCountQuery(filter2); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter2.setDeviceTypes(Arrays.asList("default0", "default1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(10, count); + countByQueryAndCheck(countQuery, 10); filter2.setRootEntity(devices.get(0).getId()); filter2.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); AssetSearchQueryFilter filter3 = new AssetSearchQueryFilter(); filter3.setRootEntity(tenantId); @@ -250,18 +253,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter3.setRelationType("Manages"); countQuery = new EntityCountQuery(filter3); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter3.setAssetTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); filter3.setRootEntity(devices.get(0).getId()); filter3.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -278,11 +277,12 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData entityDataByQuery = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData entityDataByQuery = findByQueryAndCheck(query, 5); List data = entityDataByQuery.getData(); Assert.assertEquals(data.size(), 5); data.forEach(entityData -> Assert.assertNotNull(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("phone"))); + countByQueryAndCheck(query, 5); } private void createTestUserRelations(TenantId tenantId, List users) { @@ -312,30 +312,24 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setEdgeNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setEdgeTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setEdgeTypes(List.of("default")); filter.setEdgeNameFilter("Edge1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.EDGE); entityListFilter.setEntityList(edges.stream().map(Edge::getId).map(EdgeId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); edgeService.deleteEdgesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -360,13 +354,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setRelationType("Manages"); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter.setEdgeTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); } private Edge createEdge(int i, String type) { @@ -424,11 +415,11 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 25); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(25, loadedEntities.size()); @@ -437,6 +428,9 @@ public class EntityServiceTest extends AbstractServiceTest { List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); Assert.assertEquals(deviceTemperatures, loadedTemperatures); + //count query + countByQueryAndCheck(query, 25); + pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); @@ -447,13 +441,12 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFilters = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, highTemperatures.size()); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -464,6 +457,9 @@ public class EntityServiceTest extends AbstractServiceTest { Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + //count query + countByQueryAndCheck(query, deviceHighTemperatures.size()); + deviceService.deleteDevicesByTenantId(tenantId); } @@ -482,13 +478,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(63, count); + countByQueryAndCheck(countQuery, 63); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(27, count); + countByQueryAndCheck(countQuery, 27); filter.setMultiRootEntitiesType(EntityType.ASSET); filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); @@ -496,13 +489,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(9, count); + countByQueryAndCheck(countQuery, 9); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); - } @Test @@ -538,15 +528,6 @@ public class EntityServiceTest extends AbstractServiceTest { onlineStatusFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(onlineStatusFilter); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - long expectedEntitiesCnt = entityNameByTypeMap.entrySet() .stream() .filter(e -> !e.getKey().equals("building")) @@ -554,6 +535,14 @@ public class EntityServiceTest extends AbstractServiceTest { .map(Map.Entry::getValue) .filter(e -> StringUtils.endsWith(e, "_1")) .count(); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + PageData data = findByQueryAndCheck(query, expectedEntitiesCnt); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } Assert.assertEquals(expectedEntitiesCnt, loadedEntities.size()); Map actualRelations = new HashMap<>(); @@ -603,11 +592,11 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 25); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(25, loadedEntities.size()); @@ -628,12 +617,12 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -676,11 +665,11 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "consumption")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 5); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(5, loadedEntities.size()); @@ -700,12 +689,12 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highConsumptions.size(), loadedEntities.size()); @@ -896,9 +885,7 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -906,7 +893,7 @@ public class EntityServiceTest extends AbstractServiceTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -931,7 +918,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -945,11 +932,12 @@ public class EntityServiceTest extends AbstractServiceTest { devices.get(1).setLabel(null); devices.forEach(deviceService::saveDevice); + // FIXME (for Dasha, plz investigate): + // this and other tests below submit an empty value to a KEY FILTER, this is not "search text". + // why are we supposed to ignore it and return all devices? maybe it's a bug? String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -961,9 +949,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = devices.get(2).getLabel(); EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size() - 1, result.getTotalElements()); + findByQueryAndCheck(query, devices.size() - 1); } @Test @@ -976,8 +962,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -989,9 +974,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.STARTS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1003,9 +986,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.ENDS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1017,9 +998,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1031,9 +1010,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = "label-"; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(2, result.getTotalElements()); + findByQueryAndCheck(query, 2); } @Test @@ -1045,9 +1022,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1071,34 +1046,27 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test public void testFindEntityDataByQuery_filter_entity_name_ends_with() { List devices = new ArrayList<>(); + String suffixes = RandomStringUtils.randomAlphanumeric(5); for (int i = 0; i < 10; i++) { Device device = new Device(); device.setTenantId(tenantId); - device.setName("Device " + i + " test"); + device.setName("Device " + i + suffixes); device.setType("default"); devices.add(device); } @@ -1107,29 +1075,21 @@ public class EntityServiceTest extends AbstractServiceTest { EntityNameFilter deviceTypeFilter = new EntityNameFilter(); deviceTypeFilter.setEntityType(EntityType.DEVICE); - deviceTypeFilter.setEntityNameFilter("%test"); + deviceTypeFilter.setEntityNameFilter("%" + suffixes); EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + findByQueryAndCheck(query, devices.size()); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); - - deviceTypeFilter.setEntityNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); - - deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter("%" + suffixes + "%"); + findByQueryAndCheck(query, devices.size()); - deviceTypeFilter.setEntityNameFilter("test"); + deviceTypeFilter.setEntityNameFilter(suffixes + "%"); + findByQueryAndCheck(query, 0); - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter(suffixes); + findByQueryAndCheck(query, 0); } @Test @@ -1153,19 +1113,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setEntityNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1189,24 +1143,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1239,24 +1185,6 @@ public class EntityServiceTest extends AbstractServiceTest { assertThat(deviceName).isEqualTo(devices.get(0).getName()); } - @Test - public void testFindEntitiesByApiUsageStateFilter() { - apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); - ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); - apiUsageStateFilter.setCustomerId(customerId); - - List entityFields = List.of( - new EntityKey(EntityKeyType.ENTITY_FIELD, "name") - ); - - EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); - EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); - String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(name).isEqualTo(TEST_CUSTOMER_NAME); - } - @Test public void testFindEntitiesByRelationEntityTypeFilter() { Customer customer = new Customer(); @@ -1341,24 +1269,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1382,19 +1302,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1418,24 +1332,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1459,24 +1365,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1488,6 +1386,7 @@ public class EntityServiceTest extends AbstractServiceTest { asset.setTenantId(tenantId); asset.setName("Asset test" + i); asset.setType("default"); + asset.setAssetProfileId(assetProfileService.findDefaultAssetProfile(tenantId).getId()); assets.add(asset); } @@ -1500,19 +1399,107 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("%test"); + findByQueryAndCheck(query, 0); + } + + @Test + public void testFindEntitiesBySingleEntityFilter_customer() { + List customerDevices = new ArrayList<>(); + List tenantDevices = new ArrayList<>(); + + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test" + i); + device.setType("default"); + Device saved = deviceService.saveDevice(device); + customerDevices.add(saved); + } - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Tenant test device" + i); + device.setType("default"); + tenantDevices.add(deviceService.saveDevice(device)); + } + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(customerDevices.get(0).getId()); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); + + PageData result = findByQueryAndCheck(query, 1); + String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); + + // try to find tenant device by customer user + SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); + tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); + EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); + PageData customerResults2 = entityService.findEntityDataByQuery(tenantId, customerId, customerQuery2); + + assertEquals(0, customerResults2.getTotalElements()); + + // find by tenant user with group permission + PageData results3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + + assertEquals(1, results3.getTotalElements()); + String deviceName3 = results3.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(deviceName3).isEqualTo(customerDevices.get(0).getName()); + } + + private List getResultDeviceIds(PageData result) { + return result.getData().stream().map(entityData -> (DeviceId) entityData.getEntityId()).collect(Collectors.toList()); + } + + private Device createDevice(CustomerId customerId) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test " + RandomStringUtils.randomAlphabetic(5)); + device.setType("default"); + return device; + } + + @Test + public void testFindEntitiesByApiUsageStateFilter() { + ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); + + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); + PageData result = findByQueryAndCheck(query, 1); + String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(name).isEqualTo(TEST_TENANT_NAME); + + // find by customer user with generic permissions + apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); + PageData customerResult = entityService.findEntityDataByQuery(tenantId, customerId, query); + + assertEquals(1, customerResult.getTotalElements()); + String customerResultName = customerResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(customerResultName).isEqualTo(TEST_CUSTOMER_NAME); + + // find by tenant user with customerId filter + apiUsageStateFilter.setCustomerId(customerId); + PageData tenantResult = searchEntities(query); + assertEquals(1, tenantResult.getTotalElements()); + String tenantResultName = tenantResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); } private PageData searchEntities(EntityDataQuery query) { @@ -1597,11 +1584,11 @@ public class EntityServiceTest extends AbstractServiceTest { for (EntityKeyType currentAttributeKeyType : attributesEntityTypes) { List latestValues = Collections.singletonList(new EntityKey(currentAttributeKeyType, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -1618,14 +1605,12 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFiltersHighTemperature = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersHighTemperature); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, highTemperatures.size()); loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -1718,7 +1703,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterTemperature); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, greaterTemperatures.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(greaterTemperatures.size(), loadedEntities.size()); @@ -1732,7 +1717,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, greaterOrEqualTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(greaterOrEqualTemperatures.size(), loadedEntities.size()); @@ -1746,7 +1731,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, lessTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(lessTemperatures.size(), loadedEntities.size()); @@ -1760,7 +1745,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, lessOrEqualTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(lessOrEqualTemperatures.size(), loadedEntities.size()); @@ -1774,7 +1759,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, equalTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(equalTemperatures.size(), loadedEntities.size()); @@ -1788,7 +1773,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notEqualTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notEqualTemperatures.size(), loadedEntities.size()); @@ -1843,12 +1828,12 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -1871,13 +1856,12 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFilters = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, highTemperatures.size()); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -1994,7 +1978,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, equalStrings.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(equalStrings.size(), loadedEntities.size()); @@ -2007,7 +1991,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notEqualStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notEqualStrings.size(), loadedEntities.size()); @@ -2020,7 +2004,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, startsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(startsWithStrings.size(), loadedEntities.size()); @@ -2033,7 +2017,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, endsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(endsWithStrings.size(), loadedEntities.size()); @@ -2046,7 +2030,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, containsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(containsStrings.size(), loadedEntities.size()); @@ -2059,7 +2043,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notContainsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notContainsStrings.size(), loadedEntities.size()); @@ -2072,7 +2056,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, deviceTypeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2117,7 +2101,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2132,7 +2116,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2145,7 +2129,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2158,7 +2142,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2171,7 +2155,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2184,7 +2168,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2232,7 +2216,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, deviceTypeFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2240,7 +2224,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, createdTimeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2248,7 +2232,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, null); query = new EntityDataQuery(filter, pageLink, entityFields, null, nameFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2296,12 +2280,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing EntityDataPageLink originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); EntityDataQuery originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, deviceTypeFilters); - PageData originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + PageData originalData = findByQueryAndCheck(originalQuery, expectedDevicesSize); // query without textSearch - optimization is performing EntityDataPageLink optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters); - PageData optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + PageData optimizedData = findByQueryAndCheck(optimizedQuery, expectedDevicesSize); List loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2325,12 +2309,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, attributeFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, attributeFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2354,12 +2338,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, nameFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2387,10 +2371,9 @@ public class EntityServiceTest extends AbstractServiceTest { private List getLoadedEntities(PageData data, EntityDataQuery query) { List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } return loadedEntities; @@ -2421,13 +2404,13 @@ public class EntityServiceTest extends AbstractServiceTest { private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { @@ -2436,7 +2419,7 @@ public class EntityServiceTest extends AbstractServiceTest { tsKv.setDoubleValue(value); KvEntry telemetryValue = new DoubleDataEntry(key, value); BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); - return timeseriesService.save(SYSTEM_TENANT_ID, entityId, timeseries); + return timeseriesService.save(tenantId, entityId, timeseries); } private void createMultiRootHierarchy(List buildings, List apartments, @@ -2510,4 +2493,78 @@ public class EntityServiceTest extends AbstractServiceTest { } } } + + @Test + public void testFindEntitiesWithEntityViewFilter() { + EntityView entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setCustomerId(customerId); + entityView.setName("test"); + entityView.setType("default"); + entityView.setEntityId(new DeviceId(UUID.randomUUID())); + entityView.setKeys(new TelemetryEntityView(List.of("test"), null)); + entityView.setStartTimeMs(124); + entityView.setEndTimeMs(256); + entityView.setExternalId(new EntityViewId(UUID.randomUUID())); + entityView.setAdditionalInfo(JacksonUtil.newObjectNode().put("test", "test")); + entityView = entityViewDao.save(tenantId, entityView); + + EntityViewTypeFilter entityViewTypeFilter = new EntityViewTypeFilter(); + entityViewTypeFilter.setEntityViewNameFilter("test"); + entityViewTypeFilter.setEntityViewTypes(List.of("non-existing", "default")); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataQuery query = new EntityDataQuery(entityViewTypeFilter, pageLink, entityFields, Collections.emptyList(), null); + + PageData relationsResult = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + assertThat(relationsResult.getData()).hasSize(1); + assertThat(relationsResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).isEqualTo(entityView.getName()); + + // find with non existing name + entityViewTypeFilter.setEntityViewNameFilter("non-existing"); + PageData relationsResult2 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + assertThat(relationsResult2.getData()).hasSize(0); + + // find with non existing type + entityViewTypeFilter.setEntityViewNameFilter(null); + entityViewTypeFilter.setEntityViewTypes(Collections.singletonList("non-existing")); + + PageData relationsResult3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + assertThat(relationsResult3.getData()).hasSize(0); + } + + private PageData findByQuery(EntityDataQuery query) { + return findByQuery(new CustomerId(CustomerId.NULL_UUID), query); + } + + protected PageData findByQuery(CustomerId customerId, EntityDataQuery query) { + return entityService.findEntityDataByQuery(tenantId, customerId, query); + } + + private PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { + return findByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), query, expectedResultSize); + } + + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + PageData result = entityService.findEntityDataByQuery(tenantId, customerId, query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected long countByQuery(CustomerId customerId, EntityCountQuery query) { + return entityService.countEntitiesByQuery(tenantId, customerId, query); + } + + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); + } + + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + long result = countByQuery(customerId, query); + assertThat(result).isEqualTo(expectedResult); + return result; + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 0b6e70b84c..449eca3e47 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -626,8 +626,7 @@ public class TbRuleEngineQueueConsumerManagerTest { .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); verify(consumer, times(1)).subscribe(any()); verify(consumer).subscribe(eq(expectedPartitions)); - verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() - .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList())))); + verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions))); verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); verify(consumer, never()).unsubscribe(); @@ -743,9 +742,11 @@ public class TbRuleEngineQueueConsumerManagerTest { } @Override - protected void doSubscribe(List topicNames) { - log.debug("doSubscribe({})", topicNames); - this.topics = topicNames; + protected void doSubscribe(Set partitions) { + this.topics = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toList()); + log.debug("doSubscribe({})", topics); subscribed = true; } diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index b58d19e0e5..9880ec964b 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -556,7 +556,7 @@ public class DefaultDeviceStateServiceTest { .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false)); PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of( new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi) - )); + ), Collections.emptyMap()); service.onApplicationEvent(event); Thread.sleep(100); } diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 9951caa876..ba6863c705 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -53,6 +53,9 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 -server.log_controller_error_stack_trace=false +server.log_controller_error_stack_trace=true transport.gateway.dashboard.sync.enabled=false + +queue.edqs.sync_enabled=false +queue.edqs.api_enabled=false diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 13c93da411..a0efcf52c1 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -9,7 +9,7 @@ - + diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index 103d7fe9cd..1b14ec4a23 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -26,6 +26,8 @@ public interface TbQueueRequestTemplate send(Request request, long timeoutNs); + ListenableFuture send(Request request, Integer partition); + void stop(); void setMessagesStats(MessagesStats messagesStats); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java index 641b4d8faf..bf77c8501e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -15,9 +15,17 @@ */ package org.thingsboard.server.queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + public interface TbQueueResponseTemplate { - void init(TbQueueHandler handler); + void subscribe(); + + void subscribe(Set partitions); + + void launch(TbQueueHandler handler); void stop(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java new file mode 100644 index 0000000000..c78998cd6a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -0,0 +1,102 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + +public enum ObjectType { + TENANT, + TENANT_PROFILE, + CUSTOMER, + ADMIN_SETTINGS, + QUEUE, + RPC, + RULE_CHAIN, + OTA_PACKAGE, + RESOURCE, + ROLE, + ENTITY_GROUP, + DEVICE_GROUP_OTA_PACKAGE, + GROUP_PERMISSION, + BLOB_ENTITY, + SCHEDULER_EVENT, + EVENT, + RULE_NODE, + CONVERTER, + INTEGRATION, + USER, + USER_CREDENTIALS, + USER_AUTH_SETTINGS, + EDGE, + WIDGETS_BUNDLE, + WIDGET_TYPE, + DASHBOARD, + DEVICE_PROFILE, + DEVICE, + DEVICE_CREDENTIALS, + ASSET_PROFILE, + ASSET, + ENTITY_VIEW, + ALARM, + ENTITY_ALARM, + OAUTH2_CLIENT, + OAUTH2_DOMAIN, + OAUTH2_MOBILE, + USER_SETTINGS, + NOTIFICATION_TARGET, + NOTIFICATION_TEMPLATE, + NOTIFICATION_RULE, + WHITE_LABELING, + CUSTOM_TRANSLATION, + ALARM_COMMENT, + ALARM_TYPE, + API_USAGE_STATE, + QUEUE_STATS, + + AUDIT_LOG, + RELATION, + ATTRIBUTE_KV, + LATEST_TS_KV; + + public static final Set edqsTenantTypes = EnumSet.of( + TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, + BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS + ); + public static final Set edqsTypes = new HashSet<>(edqsTenantTypes); + public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, + API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); + + static { + edqsTypes.addAll(Arrays.asList(TENANT, ENTITY_GROUP, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + } + + public EntityType toEntityType() { + return EntityType.valueOf(name()); + } + + public static ObjectType fromEntityType(EntityType entityType) { + try { + return ObjectType.valueOf(entityType.name()); + } catch (Exception e) { + return null; + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java new file mode 100644 index 0000000000..0813e72123 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class AlarmType { + + private TenantId tenantId; + private String type; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java new file mode 100644 index 0000000000..f2c35466dc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class AttributeKv implements EdqsObject { + + private EntityId entityId; + private AttributeScope scope; + private String key; + private Long version; + + private Long lastUpdateTs; // optional (on deletion) + private KvEntry value; // optional (on deletion) + + public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = attributeKvEntry.getKey(); + this.version = version; + this.lastUpdateTs = attributeKvEntry.getLastUpdateTs(); + this.value = attributeKvEntry; + } + + public AttributeKv(EntityId entityId, AttributeScope scope, String key, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = key; + this.version = version; + } + + @Override + public String key() { + return "a_" + entityId + "_" + scope + "_" + key; + } + + @Override + public Long version() { + return version; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java new file mode 100644 index 0000000000..b070121baa --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@AllArgsConstructor +@Builder +public class EdqsEvent { + + private final TenantId tenantId; + private final ObjectType objectType; + private final EdqsEventType eventType; + private final EdqsObject object; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java new file mode 100644 index 0000000000..2907691f91 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +public enum EdqsEventType { + UPDATED, + DELETED +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java new file mode 100644 index 0000000000..9a2836149a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public interface EdqsObject { + + @JsonIgnore + String key(); + + @JsonIgnore + Long version(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java new file mode 100644 index 0000000000..69bb4b8888 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +@Data +@JsonIgnoreProperties +public class EdqsSyncRequest { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java new file mode 100644 index 0000000000..decfbbd116 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.EntityIdFields; + +import java.util.UUID; + +@Data +@NoArgsConstructor +public class Entity implements EdqsObject { + + private EntityType type; + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + private EntityFields fields; + + public Entity(EntityType type) { + this.type = type; + } + + public Entity(EntityType type, EntityFields fields) { + this.type = type; + this.fields = fields; + } + + public Entity(EntityType entityType, UUID id, long version) { + this.type = entityType; + this.fields = new EntityIdFields(id, version); + } + + @Override + public String key() { + return "e_" + fields.getId().toString(); + } + + @Override + public Long version() { + return fields.getVersion(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java new file mode 100644 index 0000000000..8aaabad1a7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class LatestTsKv implements EdqsObject { + + private EntityId entityId; + private String key; + private Long version; + + private Long ts; // optional (on deletion) + private KvEntry value; // optional (on deletion) + + public LatestTsKv(EntityId entityId, TsKvEntry tsKvEntry, Long version) { + this.entityId = entityId; + this.key = tsKvEntry.getKey(); + this.ts = tsKvEntry.getTs(); + this.version = version != null ? version : 0L; + this.value = tsKvEntry; + } + + public LatestTsKv(EntityId entityId, String key, Long version) { + this.entityId = entityId; + this.key = key; + this.version = version != null ? version : 0L; + } + + public String key() { + return "l_" + entityId + "_" + key; + } + + @Override + public Long version() { + return version; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java new file mode 100644 index 0000000000..802a9c23cf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class ToCoreEdqsMsg { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java new file mode 100644 index 0000000000..e5382a43c1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ToCoreEdqsRequest { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + + @JsonIgnore + public ToCoreEdqsMsg toInternalMsg() { + return new ToCoreEdqsMsg(syncRequest, apiEnabled); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java new file mode 100644 index 0000000000..f520e60bdc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.CustomerId; + +import java.util.UUID; + +@Data +@SuperBuilder +public class AbstractEntityFields implements EntityFields { + + private UUID id; + private long createdTime; + private UUID tenantId; + private UUID customerId; + private String name; + private Long version; + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { + this.id = id; + this.createdTime = createdTime; + this.tenantId = tenantId; + this.customerId = (customerId != null && customerId != CustomerId.NULL_UUID) ? customerId : null; + this.name = name; + this.version = version; + } + + public AbstractEntityFields() { + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + this(id, createdTime, tenantId, null, name, version); + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, Long version) { + this(id, createdTime, tenantId, customerId, null, version); + + } + + public AbstractEntityFields(UUID id, long createdTime, String name, Long version) { + this(id, createdTime, null, name, version); + } + + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId) { + this(id, createdTime, tenantId, null, null, null); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java new file mode 100644 index 0000000000..f81604cfbc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@SuperBuilder +public class ApiUsageStateFields extends AbstractEntityFields { + + private EntityId entityId; + private ApiUsageStateValue transportState; + private ApiUsageStateValue dbStorageState; + private ApiUsageStateValue reExecState; + private ApiUsageStateValue jsExecState; + private ApiUsageStateValue tbelExecState; + private ApiUsageStateValue emailExecState; + private ApiUsageStateValue smsExecState; + private ApiUsageStateValue alarmExecState; + + public ApiUsageStateFields(UUID id, long createdTime, UUID tenantId, UUID entityId, String entityType, ApiUsageStateValue transportState, ApiUsageStateValue dbStorageState, + ApiUsageStateValue reExecState, ApiUsageStateValue jsExecState, ApiUsageStateValue tbelExecState, + ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState) { + super(id, createdTime, tenantId); + this.entityId = (entityType != null && entityId != null) ? EntityIdFactory.getByTypeAndUuid(entityType, entityId) : null; + this.transportState = transportState; + this.dbStorageState = dbStorageState; + this.reExecState = reExecState; + this.jsExecState = jsExecState; + this.tbelExecState = tbelExecState; + this.emailExecState = emailExecState; + this.smsExecState = smsExecState; + this.alarmExecState = alarmExecState; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java new file mode 100644 index 0000000000..c5845efb87 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetFields extends AbstractEntityFields implements ProfileAwareFields { + + private String type; + private UUID assetProfileId; + private String label; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return assetProfileId; + } + + public AssetFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, + Long version, String type, String label, UUID assetProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.assetProfileId = assetProfileId; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java new file mode 100644 index 0000000000..bbe7efb56a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public AssetProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java new file mode 100644 index 0000000000..00ab3776f8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class CustomerFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + + public CustomerFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo, + String country, String state, String city, String address, String address2, String zip, String phone, String email) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java new file mode 100644 index 0000000000..af1640b7be --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + + +@Data +@NoArgsConstructor +@SuperBuilder +public class DashboardFields extends AbstractEntityFields { + + public DashboardFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { + super(id, createdTime, tenantId, customerId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java new file mode 100644 index 0000000000..f1622b83d9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceFields extends AbstractEntityFields implements ProfileAwareFields { + + private String label; + private String type; + private UUID deviceProfileId; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return deviceProfileId; + } + + public DeviceFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, String type, + String label, UUID deviceProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.label = label; + this.type = type; + this.deviceProfileId = deviceProfileId; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java new file mode 100644 index 0000000000..5015196f69 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.DeviceProfileType; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceProfileFields extends AbstractEntityFields { + + private String type; + private boolean isDefault; + + public DeviceProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, DeviceProfileType type, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.type = type.name(); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java new file mode 100644 index 0000000000..460a43e7c8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EdgeFields extends AbstractEntityFields { + + private String type; + private String label; + private String additionalInfo; + + public EdgeFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, + String type, String label, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java new file mode 100644 index 0000000000..7536586f57 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -0,0 +1,171 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.UUID; + +public interface EntityFields { + + Logger log = LoggerFactory.getLogger(EntityFields.class); + + default UUID getId() { + return null; + } + + default UUID getTenantId() { + return null; + } + + default UUID getCustomerId() { + return null; + } + + default long getCreatedTime() { + return 0; + } + + default String getName() { + return ""; + } + + default String getType() { + return ""; + } + + default String getLabel() { + return ""; + } + + default String getAdditionalInfo() { + return ""; + } + + default String getEmail() { + return ""; + } + + default String getCountry() { + return ""; + } + + default String getState() { + return ""; + } + + default String getCity() { + return ""; + } + + default String getAddress() { + return ""; + } + + default String getAddress2() { + return ""; + } + + default String getZip() { + return ""; + } + + default String getPhone() { + return ""; + } + + default String getRegion() { + return ""; + } + + default String getFirstName() { + return ""; + } + + default String getLastName() { + return ""; + } + + default boolean isEdgeTemplate() { + return false; + } + + default String getConfiguration() { + return ""; + } + + default String getSchedule() { + return ""; + } + + default EntityId getOriginatorId() { + return null; + } + + default String getQueueName() { + return ""; + } + + default String getServiceId() { + return ""; + } + + default boolean isDefault() { + return false; + } + + default UUID getOwnerId() { + return null; + } + + default Long getVersion() { + return null; + } + + default String getAsString(String key) { + return switch (key) { + case "createdTime" -> Long.toString(getCreatedTime()); + case "type" -> getType(); + case "label" -> getLabel(); + case "additionalInfo" -> getAdditionalInfo(); + case "email" -> getEmail(); + case "country" -> getCountry(); + case "state" -> getState(); + case "city" -> getCity(); + case "address" -> getAddress(); + case "address2" -> getAddress2(); + case "zip" -> getZip(); + case "phone" -> getPhone(); + case "region" -> getRegion(); + case "firstName" -> getFirstName(); + case "lastName" -> getLastName(); + case "edgeTemplate" -> Boolean.toString(isEdgeTemplate()); + case "configuration" -> getConfiguration(); + case "schedule" -> getSchedule(); + case "originatorId" -> getOriginatorId().getId().toString(); + case "originatorType" -> getOriginatorId().getEntityType().toString(); + case "queueName" -> getQueueName(); + case "serviceId" -> getServiceId(); + default -> { + log.warn("Unknown field '{}'", key); + yield null; + } + }; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java new file mode 100644 index 0000000000..aabac7865f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityIdFields implements EntityFields { + + private UUID id; + private Long version; + + public EntityIdFields(UUID id, Long version) { + this.id = id; + this.version = version; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java new file mode 100644 index 0000000000..5635566cc4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityViewFields extends AbstractEntityFields { + + private String type; + private String additionalInfo; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java new file mode 100644 index 0000000000..5a0db392ef --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -0,0 +1,298 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.widget.WidgetType; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.util.UUID; + +public class FieldsUtil { + + public static EntityFields toFields(Object entity) { + if (entity instanceof Customer customer) { + return toFields(customer); + } else if (entity instanceof Tenant tenant) { + return toFields(tenant); + } else if (entity instanceof TenantProfile tenantProfile) { + return toFields(tenantProfile); + } else if (entity instanceof Device device) { + return toFields(device); + } else if (entity instanceof Asset asset) { + return toFields(asset); + } else if (entity instanceof Edge edge) { + return toFields(edge); + } else if (entity instanceof EntityView entityView) { + return toFields(entityView); + } else if (entity instanceof User user) { + return toFields(user); + } else if (entity instanceof Dashboard dashboard) { + return toFields(dashboard); + } else if (entity instanceof RuleChain ruleChain) { + return toFields(ruleChain); + } else if (entity instanceof RuleNode ruleNode) { + return toFields(ruleNode); + } else if (entity instanceof WidgetType widgetType) { + return toFields(widgetType); + } else if (entity instanceof WidgetsBundle widgetsBundle) { + return toFields(widgetsBundle); + } else if (entity instanceof DeviceProfile deviceProfile) { + return toFields(deviceProfile); + } else if (entity instanceof AssetProfile assetProfile) { + return toFields(assetProfile); + } else if (entity instanceof QueueStats queueStats) { + return toFields(queueStats); + } else if (entity instanceof ApiUsageState apiUsageState) { + return toFields(apiUsageState); + } else { + throw new IllegalArgumentException("Unsupported entity type: " + entity.getClass().getName()); + } + } + + private static CustomerFields toFields(Customer entity) { + return CustomerFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .version(entity.getVersion()) + .build(); + } + + private static TenantFields toFields(Tenant entity) { + return TenantFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .region(entity.getRegion()) + .version(entity.getVersion()) + .build(); + } + + private static TenantProfileFields toFields(TenantProfile tenantProfile) { + return TenantProfileFields.builder() + .id(tenantProfile.getUuidId()) + .createdTime(tenantProfile.getCreatedTime()) + .name(tenantProfile.getName()) + .isDefault(tenantProfile.isDefault()) + .build(); + } + + private static DeviceFields toFields(Device entity) { + return DeviceFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .deviceProfileId(entity.getDeviceProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static AssetFields toFields(Asset entity) { + return AssetFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .assetProfileId(entity.getAssetProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EdgeFields toFields(Edge entity) { + return EdgeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EntityViewFields toFields(EntityView entity) { + return EntityViewFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static UserFields toFields(User entity) { + return UserFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .firstName(entity.getFirstName()) + .lastName(entity.getLastName()) + .email(entity.getEmail()) + .phone(entity.getPhone()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static DashboardFields toFields(Dashboard entity) { + return DashboardFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getTitle()) + .version(entity.getVersion()) + .build(); + } + + private static RuleChainFields toFields(RuleChain entity) { + return RuleChainFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static RuleNodeFields toFields(RuleNode entity) { + return RuleNodeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .build(); + } + + private static WidgetTypeFields toFields(WidgetType entity) { + return WidgetTypeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static WidgetsBundleFields toFields(WidgetsBundle entity) { + return WidgetsBundleFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static AssetProfileFields toFields(DeviceProfile entity) { + return AssetProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static DeviceProfileFields toFields(AssetProfile entity) { + return DeviceProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .type(DeviceProfileType.DEFAULT.name()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static QueueStatsFields toFields(QueueStats entity) { + return QueueStatsFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .queueName(entity.getQueueName()) + .serviceId(entity.getServiceId()) + .build(); + } + + private static ApiUsageStateFields toFields(ApiUsageState entity) { + return ApiUsageStateFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .entityId(entity.getEntityId()) + .transportState(entity.getTransportState()) + .dbStorageState(entity.getDbStorageState()) + .reExecState(entity.getReExecState()) + .jsExecState(entity.getJsExecState()) + .tbelExecState(entity.getTbelExecState()) + .emailExecState(entity.getEmailExecState()) + .smsExecState(entity.getSmsExecState()) + .alarmExecState(entity.getAlarmExecState()) + .build(); + } + + public static String getText(JsonNode node) { + return node != null ? node.asText() : ""; + } + + private static UUID getCustomerId(CustomerId customerId) { + return (customerId != null && !customerId.getId().equals(CustomerId.NULL_UUID)) ? customerId.getId() : null; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java new file mode 100644 index 0000000000..ff3af4ee65 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class GenericFields extends AbstractEntityFields { + + private String additionalInfo; + + public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + } + + public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java new file mode 100644 index 0000000000..83134af716 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import java.util.UUID; + +public interface ProfileAwareFields extends EntityFields { + + String getProfileName(); + + UUID getProfileId(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java new file mode 100644 index 0000000000..32d74a4b39 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class QueueStatsFields extends AbstractEntityFields { + + private String queueName; + private String serviceId; + + @Override + public String getName() { + return queueName + '_' + serviceId; + } + + public QueueStatsFields(UUID id, long createdTime, UUID tenantId, String queueName, String serviceId) { + super(id, createdTime, tenantId); + this.queueName = queueName; + this.serviceId = serviceId; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java new file mode 100644 index 0000000000..ced202eeea --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleChainFields extends AbstractEntityFields { + + private String additionalInfo; + + public RuleChainFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java new file mode 100644 index 0000000000..82495a96de --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleNodeFields implements EntityFields { + + private UUID id; + private long createdTime; + private String name; + private String additionalInfo; + + public RuleNodeFields(UUID id, long createdTime, String name, JsonNode additionalInfo) { + this.id = id; + this.createdTime = createdTime; + this.name = name; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java new file mode 100644 index 0000000000..6942d5ea7b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + private String region; + + public TenantFields(UUID id, long createdTime, String name, Long version, + JsonNode additionalInfo, String country, String state, String city, String address, + String address2, String zip, String phone, String email, String region) { + super(id, createdTime, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + this.region = region; + } + + public TenantFields(UUID id, Long version) { + super(id, 0L, null, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java new file mode 100644 index 0000000000..766a160fc2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public TenantProfileFields(UUID id, long createdTime, String name, boolean isDefault) { + super(id, createdTime, TenantId.SYS_TENANT_ID.getId(), null, name, 0L); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java new file mode 100644 index 0000000000..98e0efecc2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class UserFields extends AbstractEntityFields { + + private String firstName; + private String lastName; + private String email; + private String phone; + private String additionalInfo; + + public UserFields(UUID id, long createdTime, UUID tenantId, UUID customerId, + Long version, String firstName, String lastName, String email, + String phone, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, version); + this.firstName = firstName; + this.lastName = lastName; + this.email = email; + this.phone = phone; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java new file mode 100644 index 0000000000..330199d8b2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetTypeFields extends AbstractEntityFields { + + public WidgetTypeFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java new file mode 100644 index 0000000000..88d9a1a96c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetsBundleFields extends AbstractEntityFields { + + public WidgetsBundleFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java new file mode 100644 index 0000000000..135061a842 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import com.fasterxml.jackson.annotation.JsonIncludeProperties; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EdqsRequest { + + private EntityDataQuery entityDataQuery; + private EntityCountQuery entityCountQuery; + @JsonIncludeProperties({"genericPermissions", "groupPermissions"}) + private MergedUserPermissions userPermissions; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java new file mode 100644 index 0000000000..bcb242e083 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class EdqsResponse { + + private PageData entityDataQueryResult; + private Long entityCountQueryResult; + private String error; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java new file mode 100644 index 0000000000..0c45812690 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; + +import java.util.Collections; +import java.util.Map; + +@Data +@RequiredArgsConstructor +public class QueryResult { + + private final EntityId entityId; + private final boolean readAttrs; + private final boolean readTs; + private final Map> latest; + + public EntityData toOldEntityData() { + return new EntityData(entityId, readAttrs, readTs, latest, Collections.emptyMap(), Collections.emptyMap()); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java index 1b462db4da..cd1fdf9aa6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java @@ -15,11 +15,15 @@ */ package org.thingsboard.server.common.data.id; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.UUID; public class UserAuthSettingsId extends UUIDBased { - public UserAuthSettingsId(UUID id) { + @JsonCreator + public UserAuthSettingsId(@JsonProperty("id") UUID id) { super(id); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java index 8d46a46781..625a1d417c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java @@ -23,6 +23,7 @@ import java.util.NoSuchElementException; public abstract class BasePageDataIterable implements Iterable, Iterator { private final int fetchSize; + private SortOrder sortOrder; private List currentItems; private int currentIdx; @@ -35,6 +36,12 @@ public abstract class BasePageDataIterable implements Iterable, Iterator iterator() { return this; @@ -43,7 +50,7 @@ public abstract class BasePageDataIterable implements Iterable, Iterator extends BasePageDataIterable { this.function = function; } + public PageDataIterable(FetchFunction function, int fetchSize, SortOrder sortOrder) { + super(fetchSize, sortOrder); + this.function = function; + } + @Override PageData fetchPageData(PageLink link) { return function.fetch(link); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java similarity index 77% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java rename to common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java index 579c5bfa92..b377d63ee4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.query; +package org.thingsboard.server.common.data.permission; import lombok.AllArgsConstructor; import lombok.Getter; @@ -21,8 +21,12 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + @AllArgsConstructor -public class QuerySecurityContext { +public class QueryContext { @Getter private final TenantId tenantId; @@ -33,7 +37,11 @@ public class QuerySecurityContext { @Getter private final boolean ignorePermissionCheck; - public QuerySecurityContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { + @Getter + private final Map relatedParentIdMap = new HashMap<>(); + + public QueryContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { this(tenantId, customerId, entityType, false); } + } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java index 473fc867fa..0b9deebfd9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.common.data.query; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.validation.NoXss; @@ -26,7 +25,6 @@ import java.io.Serializable; @RequiredArgsConstructor public class DynamicValue implements Serializable { - @JsonIgnore private T resolvedValue; private final DynamicValueSourceType sourceType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index 3dd24147e7..8c5d35f03d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.query; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.ToString; @@ -24,6 +25,7 @@ import java.util.List; @Schema @ToString +@JsonIgnoreProperties(ignoreUnknown = true) public class EntityCountQuery { @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java index 79730dc23a..fe9f0d3f68 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java @@ -16,20 +16,22 @@ package org.thingsboard.server.common.data.query; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; -import lombok.RequiredArgsConstructor; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; import java.util.Map; @Data -@RequiredArgsConstructor +@AllArgsConstructor +@NoArgsConstructor public class EntityData { - private final EntityId entityId; - private final Map> latest; - private final Map timeseries; - private final Map aggLatest; + private EntityId entityId; + private Map> latest; + private Map timeseries; + private Map aggLatest; public EntityData(EntityId entityId, Map> latest, Map timeseries) { this(entityId, latest, timeseries, null); @@ -44,4 +46,5 @@ public class EntityData { aggLatest.clear(); } } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java index 383c613a38..17a712d838 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java @@ -15,10 +15,22 @@ */ package org.thingsboard.server.common.data.queue; +import lombok.Data; + public interface QueueConfig { boolean isConsumerPerPartition(); int getPollInterval(); + static QueueConfig of(boolean consumerPerPartition, long pollInterval) { + return new BasicQueueConfig(consumerPerPartition, (int) pollInterval); + } + + @Data + class BasicQueueConfig implements QueueConfig { + private final boolean consumerPerPartition; + private final int pollInterval; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java index 6c648daf02..9c3536dea1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java @@ -18,13 +18,15 @@ package org.thingsboard.server.common.data.queue; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasEntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; @EqualsAndHashCode(callSuper = true) @Data -public class QueueStats extends BaseData implements HasTenantId { +public class QueueStats extends BaseData implements HasTenantId, HasEntityType { private TenantId tenantId; private String queueName; private String serviceId; @@ -36,4 +38,8 @@ public class QueueStats extends BaseData implements HasTenantId { super(id); } + @Override + public EntityType getEntityType() { + return EntityType.QUEUE_STATS; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 62b5caafd3..927d592cd7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -25,6 +25,7 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.validation.Length; @@ -34,7 +35,7 @@ import java.io.Serializable; @Schema @EqualsAndHashCode(exclude = "additionalInfoBytes") @ToString(exclude = {"additionalInfoBytes"}) -public class EntityRelation implements HasVersion, Serializable { +public class EntityRelation implements HasVersion, Serializable, EdqsObject { private static final long serialVersionUID = 2807343040519543363L; @@ -107,7 +108,7 @@ public class EntityRelation implements HasVersion, Serializable { return typeGroup; } - @Schema(description = "Additional parameters of the relation",implementation = com.fasterxml.jackson.databind.JsonNode.class) + @Schema(description = "Additional parameters of the relation", implementation = com.fasterxml.jackson.databind.JsonNode.class) public JsonNode getAdditionalInfo() { return BaseDataWithAdditionalInfo.getJson(() -> additionalInfo, () -> additionalInfoBytes); } @@ -116,4 +117,14 @@ public class EntityRelation implements HasVersion, Serializable { BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes); } + @JsonIgnore + public String key() { + return "r_" + from + "_" + to + "_" + typeGroup + "_" + type; + } + + @Override + public Long version() { + return version; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index e89f3c1ad8..2fc3f4c040 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.util; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -75,4 +76,11 @@ public class CollectionsUtil { return isEmpty(collection) || collection.contains(element); } + public static HashSet concat(Set set1, Set set2) { + HashSet result = new HashSet<>(); + result.addAll(set1); + result.addAll(set2); + return result; + } + } diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml new file mode 100644 index 0000000000..40e89acee6 --- /dev/null +++ b/common/edqs/pom.xml @@ -0,0 +1,97 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0PE-SNAPSHOT + common + + org.thingsboard.common + edqs + jar + + Thingsboard Server EDQS API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.rocksdb + rocksdbjni + + + org.thingsboard.common + proto + + + org.thingsboard.common + data + + + org.thingsboard.common + util + + + org.thingsboard.common + message + + + org.thingsboard.common + stats + + + org.thingsboard.common + cluster-api + + + org.thingsboard.common + queue + + + org.apache.kafka + kafka-clients + + + com.github.ben-manes.caffeine + caffeine + + + org.springframework + spring-context-support + + + org.springframework.boot + spring-boot-autoconfigure + + + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + + diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java new file mode 100644 index 0000000000..c1ce21507e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class ApiUsageStateData extends BaseEntityData { + + public ApiUsageStateData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.API_USAGE_STATE; + } + + @Override + public String getEntityName() { + return getEntityOwnerName(); + } + + @Override + public String getEntityOwnerName() { + return repo.getOwnerName(fields.getEntityId()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java new file mode 100644 index 0000000000..ce2a63a2ec --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class AssetData extends ProfileAwareData { + + public AssetData(UUID id) { + super(id); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java new file mode 100644 index 0000000000..11c8b269b9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java @@ -0,0 +1,180 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString +public abstract class BaseEntityData implements EntityData { + + @Getter + private final UUID id; + @Getter + protected final Map serverAttrMap; + @Getter + private final Map tMap; + + @Getter + @Setter + private volatile UUID customerId; + + @Setter + protected TenantRepo repo; + + @Getter + @Setter + protected volatile T fields; + + public BaseEntityData(UUID id) { + this.id = id; + this.serverAttrMap = new ConcurrentHashMap<>(); + this.tMap = new ConcurrentHashMap<>(); + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE, SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + default -> null; + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return serverAttrMap.put(keyId, value) == null; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return serverAttrMap.remove(keyId) != null; + } + + @Override + public DataPoint getTs(Integer keyId) { + return tMap.get(keyId); + } + + @Override + public boolean putTs(Integer keyId, DataPoint value) { + return tMap.put(keyId, value) == null; + } + + @Override + public boolean removeTs(Integer keyId) { + return tMap.remove(keyId) != null; + } + + @Override + public EntityType getOwnerType() { + return customerId != null ? EntityType.CUSTOMER : EntityType.TENANT; + } + + @Override + public DataPoint getDataPoint(DataKey key, QueryContext ctx) { + return switch (key.type()) { + case TIME_SERIES -> getTs(key.keyId()); + case ATTRIBUTE, SERVER_ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE -> getAttr(key.keyId(), key.type()); + case ENTITY_FIELD -> getField(key, ctx); + default -> throw new RuntimeException(key.type() + " not supported"); + }; + } + + private DataPoint getField(DataKey newKey, QueryContext ctx) { + if (fields == null) { + return null; + } + String key = newKey.key(); + return switch (key) { + case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime()); + case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate()); + case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx)); + default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false); + }; + } + + @Override + public String getField(String name) { + if (fields == null) { + return null; + } + return switch (name) { + case "name" -> getEntityName(); + case "ownerName" -> getEntityOwnerName(); + case "ownerType" -> customerId != null ? EntityType.CUSTOMER.name() : EntityType.TENANT.name(); + case "entityType" -> Optional.ofNullable(getEntityType()).map(EntityType::name).orElse(""); + default -> fields.getAsString(name); + }; + } + + public String getEntityOwnerName() { + return repo.getOwnerName(getCustomerId() == null || CustomerId.NULL_UUID.equals(getCustomerId()) ? null : + new CustomerId(getCustomerId())); + } + + public String getEntityName() { + return getFields().getName(); + } + + private String getRelatedParentId(QueryContext ctx) { + return Optional.ofNullable(ctx.getRelatedParentIdMap().get(getId())) + .map(UUID::toString) + .orElse(""); + } + + @Override + public EntityType getEntityType() { + return null; + } + + @Override + public boolean isEmpty() { + return fields == null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BaseEntityData that = (BaseEntityData) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java new file mode 100644 index 0000000000..15e2573b81 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; + +import java.util.Collection; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class CustomerData extends BaseEntityData { + + private final ConcurrentMap>> entitiesById = new ConcurrentHashMap<>(); + + public CustomerData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + + public Collection> getEntities(EntityType entityType) { + var map = entitiesById.get(entityType); + if (map == null) { + return Collections.emptyList(); + } else { + return map.values(); + } + } + + public void addOrUpdate(EntityData ed) { + entitiesById.computeIfAbsent(ed.getEntityType(), et -> new ConcurrentHashMap<>()).put(ed.getId(), ed); + } + + public boolean remove(EntityData ed) { + var map = entitiesById.get(ed.getEntityType()); + if (map != null) { + return map.remove(ed.getId()) != null; + } else { + return false; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java new file mode 100644 index 0000000000..42042b2a9f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.DataPoint; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString(callSuper = true) +public class DeviceData extends ProfileAwareData { + + private final Map clientAttrMap; + private final Map sharedAttrMap; + + public DeviceData(UUID entityId) { + super(entityId); + this.clientAttrMap = new ConcurrentHashMap<>(); + this.sharedAttrMap = new ConcurrentHashMap<>(); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE -> getAttributeDataPoint(keyId); + case SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + case CLIENT_ATTRIBUTE -> clientAttrMap.get(keyId); + case SHARED_ATTRIBUTE -> sharedAttrMap.get(keyId); + default -> throw new RuntimeException(entityKeyType + " not implemented"); + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.put(keyId, value) == null; + case CLIENT_SCOPE -> clientAttrMap.put(keyId, value) == null; + case SHARED_SCOPE -> sharedAttrMap.put(keyId, value) == null; + }; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.remove(keyId) != null; + case CLIENT_SCOPE -> clientAttrMap.remove(keyId) != null; + case SHARED_SCOPE -> sharedAttrMap.remove(keyId) != null; + }; + } + + private DataPoint getAttributeDataPoint(Integer keyId) { + DataPoint dp = serverAttrMap.get(keyId); + if (dp == null) { + dp = sharedAttrMap.get(keyId); + if (dp == null) { + dp = clientAttrMap.get(keyId); + } + } + return dp; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java new file mode 100644 index 0000000000..c373baf305 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; + +public interface EntityData { + + UUID getId(); + + EntityType getEntityType(); + + UUID getCustomerId(); + + void setCustomerId(UUID customerId); + + void setRepo(TenantRepo repo); + + T getFields(); + + void setFields(T fields); + + DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType); + + boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value); + + boolean removeAttr(Integer keyId, AttributeScope scope); + + DataPoint getTs(Integer keyId); + + boolean putTs(Integer keyId, DataPoint value); + + boolean removeTs(Integer keyId); + + EntityType getOwnerType(); + + DataPoint getDataPoint(DataKey key, QueryContext queryContext); + + String getField(String name); + + boolean isEmpty(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java new file mode 100644 index 0000000000..d956519cc4 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields; + +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class EntityGroupData extends BaseEntityData { + + private final ConcurrentMap> entitiesById = new ConcurrentHashMap<>(); + + public EntityGroupData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_GROUP; + } + + public Collection> getEntities() { + return entitiesById.values(); + } + + public boolean addOrUpdate(EntityData ed) { + return entitiesById.put(ed.getId(), ed) == null; + } + + public boolean remove(EntityData ed) { + return entitiesById.remove(ed.getId()) != null; + } + + public EntityData getEntity(UUID entityId) { + return entitiesById.get(entityId); + } + + public boolean remove(UUID toId) { + return entitiesById.remove(toId) != null; + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java new file mode 100644 index 0000000000..6057ac17e7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class EntityProfileData extends BaseEntityData { + + private final EntityType entityType; + + public EntityProfileData(UUID entityId, EntityType entityType) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java new file mode 100644 index 0000000000..8fd8924b5c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class GenericData extends BaseEntityData { + + private final EntityType entityType; + + public GenericData(EntityType entityType, UUID entityId) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java new file mode 100644 index 0000000000..f98c8e6d9b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.edqs.fields.ProfileAwareFields; + +import java.util.UUID; + +public abstract class ProfileAwareData extends BaseEntityData { + + public ProfileAwareData(UUID id) { + super(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java new file mode 100644 index 0000000000..3a660af813 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; + +import java.util.UUID; + +public record RelationData(UUID fromId, EntityType fromType, UUID toId, EntityType toType, String type, + RelationTypeGroup typeGroup) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java new file mode 100644 index 0000000000..e479c94d00 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.Data; + +@Data +public class RelationInfo { + + private final String type; + private final EntityData target; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java new file mode 100644 index 0000000000..92cf10e1d0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@NoArgsConstructor +public class RelationsRepo { + + private final ConcurrentMap> fromRelations = new ConcurrentHashMap<>(); + private final ConcurrentMap> toRelations = new ConcurrentHashMap<>(); + + public boolean add(EntityData from, EntityData to, String type) { + boolean addedFromRelation = fromRelations.computeIfAbsent(from.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, to)); + boolean addedToRelation = toRelations.computeIfAbsent(to.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, from)); + return addedFromRelation || addedToRelation; + } + + public Set getFrom(UUID entityId) { + var result = fromRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public Set getTo(UUID entityId) { + var result = toRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public boolean remove(UUID from, UUID to, String type) { + boolean removedFromRelation = false; + boolean removedToRelation = false; + Set fromRelations = this.fromRelations.get(from); + if (fromRelations != null) { + removedFromRelation = fromRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(to) && relationInfo.getType().equals(type)); + } + Set toRelations = this.toRelations.get(to); + if (toRelations != null) { + removedToRelation = toRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(from) && relationInfo.getType().equals(type)); + } + return removedFromRelation || removedToRelation; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java new file mode 100644 index 0000000000..a9034e251c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; + +import java.util.UUID; + +public class TenantData extends BaseEntityData { + + public TenantData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java new file mode 100644 index 0000000000..5eb16f72fa --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public abstract class AbstractDataPoint implements DataPoint { + + @Getter + private final long ts; + + @Override + public String getStr() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public long getLong() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public double getDouble() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public boolean getBool() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public String getJson() { + throw new RuntimeException(NOT_SUPPORTED); + } + + public String toString() { + return valueToString(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java new file mode 100644 index 0000000000..ccc97a2f84 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class BoolDataPoint extends AbstractDataPoint { + + @Getter + private final boolean value; + + public BoolDataPoint(long ts, boolean value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.BOOLEAN; + } + + @Override + public boolean getBool() { + return value; + } + + @Override + public String valueToString() { + return Boolean.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java new file mode 100644 index 0000000000..b10f3badf0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import org.thingsboard.server.common.data.kv.DataType; + +public class CompressedJsonDataPoint extends CompressedStringDataPoint { + + public CompressedJsonDataPoint(long ts, String value) { + super(ts, value); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java new file mode 100644 index 0000000000..b679c94f4e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.repo.TbBytePool; +import org.xerial.snappy.Snappy; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class CompressedStringDataPoint extends AbstractDataPoint { + + public static final int MIN_STR_SIZE_TO_COMPRESS = 512; + @Getter + private final byte[] value; + + public static final AtomicInteger cnt = new AtomicInteger(); + public static final AtomicLong uncompressedLength = new AtomicLong(); + public static final AtomicLong compressedLength = new AtomicLong(); + + @SneakyThrows + public CompressedStringDataPoint(long ts, String value) { + super(ts); + cnt.incrementAndGet(); + uncompressedLength.addAndGet(value.getBytes(StandardCharsets.UTF_8).length); + this.value = TbBytePool.intern(Snappy.compress(value)); + compressedLength.addAndGet(this.value.length); + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @SneakyThrows + @Override + public String getStr() { + return Snappy.uncompressString(value); + } + + @SneakyThrows + @Override + public String valueToString() { + return Snappy.uncompressString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java new file mode 100644 index 0000000000..3871bca114 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import org.thingsboard.server.common.data.kv.DataType; + +public interface DataPoint { + + String NOT_SUPPORTED = "Not supported!"; + + long getTs(); + + DataType getType(); + + String getStr(); + + long getLong(); + + double getDouble(); + + boolean getBool(); + + String getJson(); + + String valueToString(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java new file mode 100644 index 0000000000..a7dcdf9e82 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class DoubleDataPoint extends AbstractDataPoint { + + @Getter + private final double value; + + public DoubleDataPoint(long ts, double value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.DOUBLE; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Double.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java new file mode 100644 index 0000000000..df318b6430 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.repo.TbStringPool; + +public class JsonDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public JsonDataPoint(long ts, String value) { + super(ts); + this.value = TbStringPool.intern(value); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + + @Override + public String getJson() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java new file mode 100644 index 0000000000..ac54f19707 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class LongDataPoint extends AbstractDataPoint { + + @Getter + private final long value; + + public LongDataPoint(long ts, long value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.LONG; + } + + @Override + public long getLong() { + return value; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Long.toString(value); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java new file mode 100644 index 0000000000..e1e2b5d0ef --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.repo.TbStringPool; + +public class StringDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public StringDataPoint(long ts, String value) { + this(ts, value, true); + } + + public StringDataPoint(long ts, String value, boolean deduplicate) { + super(ts); + this.value = deduplicate ? TbStringPool.intern(value) : value; + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @Override + public String getStr() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java new file mode 100644 index 0000000000..d59d0cc671 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.load; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.RandomStringUtils; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.edqs.processor.EdqsConverter; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +@RequiredArgsConstructor +public class TenantRepoLoader { + + private static final int DEVICE_COUNT = 100000; + private static final int ATTRS_PER_DEVICE = 30; + private static final int ATTRS_AVG_STR_LENGTH = 12; + private static final int ATTRS_AVG_JSON_LENGTH = 265; + private static final int TS_PER_DEVICE = 29; + private static final int TS_AVG_STR_LENGTH = 59; + private static final int TS_AVG_JSON_LENGTH = 4005; + + private static final Map ATTR_CHANCES = new HashMap<>(); + private static final Random random = new Random(); + + static { + ATTR_CHANCES.put(DataType.BOOLEAN, 5); + ATTR_CHANCES.put(DataType.STRING, 49); + ATTR_CHANCES.put(DataType.LONG, 34); + ATTR_CHANCES.put(DataType.DOUBLE, 2); + ATTR_CHANCES.put(DataType.JSON, 10); + } + + private static final Map TS_CHANCES = new HashMap<>(); + + static { + TS_CHANCES.put(DataType.BOOLEAN, 6); + TS_CHANCES.put(DataType.STRING, 19); + TS_CHANCES.put(DataType.LONG, 36); + TS_CHANCES.put(DataType.DOUBLE, 32); + TS_CHANCES.put(DataType.JSON, 7); + } + + + @Getter + private final TenantRepo tenantRepo; + + public void load() { + long ts = System.currentTimeMillis() - DEVICE_COUNT; + for (int i = 0; i < DEVICE_COUNT; i++) { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setCreatedTime(ts + i); + device.setName("Device " + i); + device.setLabel("Device Label" + i); + device.setType("Device Type " + (i % 100)); + tenantRepo.addOrUpdate(EdqsConverter.toEntity(EntityType.DEVICE, device)); + for (int j = 0; j < ATTRS_PER_DEVICE; j++) { + String key = getRandomKey(); + AttributeKv attributeKv = new AttributeKv(); + attributeKv.setEntityId(deviceId); + attributeKv.setScope(AttributeScope.SERVER_SCOPE); + attributeKv.setKey(key); + attributeKv.setLastUpdateTs(ts); + attributeKv.setValue(getRandomKvEntry(key, ATTR_CHANCES, ATTRS_AVG_STR_LENGTH, ATTRS_AVG_JSON_LENGTH)); + tenantRepo.addOrUpdateAttribute(attributeKv); + } + for (int j = 0; j < TS_PER_DEVICE; j++) { + String key = getRandomKey(); + LatestTsKv latestTsKv = new LatestTsKv(); + latestTsKv.setEntityId(deviceId); + latestTsKv.setKey(key); + latestTsKv.setTs(ts); + latestTsKv.setValue(getRandomKvEntry(key, TS_CHANCES, TS_AVG_STR_LENGTH, TS_AVG_JSON_LENGTH)); + tenantRepo.addOrUpdateLatestKv(latestTsKv); + } + } + } + + private KvEntry getRandomKvEntry(String key, Map chances, int strLength, int jsnLength) { + int i = random.nextInt(100); + int s = 0; + for (var pair : chances.entrySet()) { + s += pair.getValue(); + if (i < s) { + switch (pair.getKey()) { + case BOOLEAN -> { + return new BooleanDataEntry(key, random.nextBoolean()); + } + case LONG -> { + return new LongDataEntry(key, random.nextLong()); + } + case DOUBLE -> { + return new DoubleDataEntry(key, random.nextDouble()); + } + case STRING -> { + return new StringDataEntry(key, StringUtils.randomAlphanumeric(strLength)); + } + case JSON -> { + return new JsonDataEntry(key, StringUtils.randomAlphanumeric(jsnLength)); + } + } + } + } + throw new RuntimeException("Something went wrong"); + } + + private String getRandomKey() { + return RandomStringUtils.randomAlphabetic(10); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java new file mode 100644 index 0000000000..125f6c49fd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java @@ -0,0 +1,183 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.FieldsUtil; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +@Service +public class EdqsConverter { + + private final Map> converters = new HashMap<>(); + private final Converter defaultConverter = new JsonConverter<>(Entity.class); + + { + converters.put(ObjectType.RELATION, new JsonConverter<>(EntityRelation.class)); + converters.put(ObjectType.ATTRIBUTE_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, AttributeKv attributeKv) { + // TODO: some attributes may not fit into kafka + var proto = TransportProtos.AttributeKvProto.newBuilder() + .setEntityIdMSB(attributeKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(attributeKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(attributeKv.getEntityId().getEntityType())) + .setScope(TransportProtos.AttributeScopeProto.forNumber(attributeKv.getScope().ordinal())) + .setKey(attributeKv.getKey()) + .setVersion(attributeKv.getVersion()); + if (attributeKv.getLastUpdateTs() != null) { + proto.setLastUpdateTs(attributeKv.getLastUpdateTs()); + } + if (attributeKv.getValue() != null) { + proto.setValue(KvProtoUtil.toKeyValueTypeProto(attributeKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public AttributeKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.AttributeKvProto proto = TransportProtos.AttributeKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + AttributeScope scope = AttributeScope.values()[proto.getScope().getNumber()]; + KvEntry value = proto.hasValue() ? KvProtoUtil.fromTsKvProto(proto.getValue()) : null; + return AttributeKv.builder() + .entityId(entityId) + .scope(scope) + .key(proto.getKey()) + .version(proto.getVersion()) + .lastUpdateTs(proto.getLastUpdateTs()) + .value(value) + .build(); + } + }); + converters.put(ObjectType.LATEST_TS_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, LatestTsKv latestTsKv) { + var proto = TransportProtos.LatestTsKvProto.newBuilder() + .setEntityIdMSB(latestTsKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(latestTsKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(latestTsKv.getEntityId().getEntityType())) + .setKey(latestTsKv.getKey()) + .setVersion(latestTsKv.getVersion()); + if (latestTsKv.getTs() != null) { + proto.setTs(latestTsKv.getTs()); + } + if (latestTsKv.getValue() != null) { + proto.setValue(KvProtoUtil.toKeyValueTypeProto(latestTsKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public LatestTsKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.LatestTsKvProto proto = TransportProtos.LatestTsKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + KvEntry value = proto.hasValue() ? KvProtoUtil.fromTsKvProto(proto.getValue()) : null; + return LatestTsKv.builder() + .entityId(entityId) + .key(proto.getKey()) + .ts(proto.getTs()) + .version(proto.getVersion()) + .value(value) + .build(); + } + }); + } + + public static Entity toEntity(EntityType entityType, Object entity) { + Entity edqsEntity = new Entity(); + edqsEntity.setType(entityType); + edqsEntity.setFields(FieldsUtil.toFields(entity)); + return edqsEntity; + } + + public EdqsObject check(ObjectType type, Object object) { + if (object instanceof EdqsObject edqsObject) { + return edqsObject; + } else { + return toEntity(type.toEntityType(), object); + } + } + + @SuppressWarnings("unchecked") + @SneakyThrows + public byte[] serialize(ObjectType type, T value) { + Converter converter = (Converter) converters.get(type); + if (converter != null) { + return converter.serialize(type, value); + } else { + return defaultConverter.serialize(type, (Entity) value); + } + } + + @SneakyThrows + public EdqsObject deserialize(ObjectType type, byte[] bytes) { + Converter converter = converters.get(type); + if (converter != null) { + return converter.deserialize(type, bytes); + } else { + return defaultConverter.deserialize(type, bytes); + } + } + + @RequiredArgsConstructor + private static class JsonConverter implements Converter { + + private final Class type; + + @Override + public byte[] serialize(ObjectType objectType, T value) { + return JacksonUtil.writeValueAsBytes(value); + } + + @Override + public T deserialize(ObjectType objectType, byte[] bytes) { + return JacksonUtil.fromBytes(bytes, this.type); + } + + } + + private interface Converter { + + byte[] serialize(ObjectType type, T value) throws Exception; + + T deserialize(ObjectType type, byte[] bytes) throws Exception; + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java new file mode 100644 index 0000000000..5383311e1f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -0,0 +1,266 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.state.EdqsStateService; +import org.thingsboard.server.edqs.repo.EdqRepository; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.edqs.EdqsComponent; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; + +@EdqsComponent +@Service +@RequiredArgsConstructor +@Slf4j +public class EdqsProcessor implements TbQueueHandler, TbProtoQueueMsg> { + + private final EdqsQueueFactory queueFactory; + private final EdqsConverter converter; + private final EdqRepository repository; + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + @Autowired @Lazy + private EdqsStateService stateService; + + private MainQueueConsumerManager, QueueConfig> eventsConsumer; + private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; + + private ExecutorService consumersExecutor; + private ExecutorService mgmtExecutor; + private ScheduledExecutorService scheduler; + private ListeningExecutorService requestExecutor; + + private final VersionsStore versionsStore = new VersionsStore(); + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); + mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-mgmt"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); + requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); + + eventsConsumer = MainQueueConsumerManager., QueueConfig>builder() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) + .config(QueueConfig.of(true, config.getPollInterval())) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + process(msg, EdqsQueue.EVENTS); + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS)) + .consumerExecutor(consumersExecutor) + .taskExecutor(mgmtExecutor) + .scheduler(scheduler) + .build(); + responseTemplate = queueFactory.createEdqsResponseTemplate(); + } + + @AfterStartUp(order = 1) + public void start() { + responseTemplate.launch(this); + } + + @EventListener + public void onPartitionsChange(PartitionChangeEvent event) { + if (event.getServiceType() != ServiceType.EDQS) { + return; + } + consumersExecutor.submit(() -> { + try { + Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); + Set partitions = newPartitions.stream() + .map(tpi -> tpi.withUseInternalPartition(true)) + .collect(Collectors.toSet()); + + try { + stateService.restore(withTopic(partitions, EdqsQueue.STATE.getTopic())); // blocks until restored + } catch (Exception e) { + log.error("Failed to process restore for partitions {}", partitions, e); + } + eventsConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic())); + responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); + + Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); + Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() + .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); + if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + } + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId); + return partition != null && removedPartitions.contains(partition); + }); + } catch (Throwable t) { + log.error("Failed to handle partition change event {}", event, t); + } + }); + } + + @Override + public ListenableFuture> handle(TbProtoQueueMsg queueMsg) { + ToEdqsMsg toEdqsMsg = queueMsg.getValue(); + return requestExecutor.submit(() -> { + EdqsResponse response = new EdqsResponse(); + try { + EdqsRequest request = JacksonUtil.fromString(toEdqsMsg.getRequestMsg().getValue(), EdqsRequest.class); + TenantId tenantId = getTenantId(toEdqsMsg); + CustomerId customerId = getCustomerId(toEdqsMsg); + log.info("[{}] Handling request: {}", tenantId, request); + + if (request.getEntityDataQuery() != null) { + PageData result = repository.findEntityDataByQuery(tenantId, customerId, + request.getUserPermissions(), request.getEntityDataQuery(), false); + response.setEntityDataQueryResult(result.mapData(QueryResult::toOldEntityData)); + } else if (request.getEntityCountQuery() != null) { + long result = repository.countEntitiesByQuery(tenantId, customerId, request.getUserPermissions(), request.getEntityCountQuery(), tenantId.isSysTenantId()); + response.setEntityCountQueryResult(result); + } + + log.info("Answering with response: {}", response); + } catch (Throwable e) { + response.setError(ExceptionUtils.getStackTrace(e)); // TODO: return only the message + log.info("Answering with error", e); + } + return new TbProtoQueueMsg<>(queueMsg.getKey(), FromEdqsMsg.newBuilder() + .setResponseMsg(TransportProtos.EdqsResponseMsg.newBuilder() + .setValue(JacksonUtil.toString(response)) + .build()) + .build(), queueMsg.getHeaders()); + }); + } + + public void process(ToEdqsMsg edqsMsg, EdqsQueue queue) { + if (edqsMsg.hasEventMsg()) { + EdqsEventMsg eventMsg = edqsMsg.getEventMsg(); + TenantId tenantId = getTenantId(edqsMsg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + String key = eventMsg.getKey(); + Long version = eventMsg.hasVersion() ? eventMsg.getVersion() : null; + + if (version != null) { + if (!versionsStore.isNew(key, version)) { + return; + } + } else { + log.warn("[{}] {} doesn't have version: {}", tenantId, objectType, edqsMsg); + } + if (queue != EdqsQueue.STATE) { + stateService.save(tenantId, objectType, key, eventType, edqsMsg); + } + + EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray()); + log.info("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + + EdqsEvent event = EdqsEvent.builder() + .tenantId(tenantId) + .objectType(objectType) + .eventType(eventType) + .object(object) + .build(); + repository.processEvent(event); + } + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + private CustomerId getCustomerId(ToEdqsMsg edqsMsg) { + if (edqsMsg.getCustomerIdMSB() != 0 && edqsMsg.getCustomerIdLSB() != 0) { + return new CustomerId(new UUID(edqsMsg.getCustomerIdMSB(), edqsMsg.getCustomerIdLSB())); + } else { + return null; + } + } + + private Set withTopic(Set partitions, String topic) { + return partitions.stream() + .map(tpi -> tpi.withTopic(topic)) + .collect(Collectors.toSet()); + } + + @PreDestroy + public void destroy() throws InterruptedException { + eventsConsumer.stop(); + eventsConsumer.awaitStop(); + responseTemplate.stop(); + + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + scheduler.shutdownNow(); + requestExecutor.shutdownNow(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java new file mode 100644 index 0000000000..762e5b37ec --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; + +@Slf4j +public class EdqsProducer { + + private final EdqsQueue queue; + private final EdqsPartitionService partitionService; + + private final TbQueueProducer> producer; + + @Builder + public EdqsProducer(EdqsQueue queue, + EdqsPartitionService partitionService, + TbQueueProducer> producer) { + this.queue = queue; + this.partitionService = partitionService; + this.producer = producer; + } + + // TODO: queue prefix! + + public void send(TenantId tenantId, ObjectType type, String key, ToEdqsMsg msg) { + String topic = queue.getTopic(); + TbQueueCallback callback = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); // fixme log levels + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}][{}] Failed to publish msg to {}: {}", tenantId, type, key, topic, msg, t); + } + }; + if (producer instanceof TbKafkaProducerTemplate> kafkaProducer) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .partition(partitionService.resolvePartition(tenantId)) + .useInternalPartition(true) + .build(); + kafkaProducer.send(tpi, key, new TbProtoQueueMsg<>(null, msg), callback); // specifying custom key for compaction + } else { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .build(); + producer.send(tpi, new TbProtoQueueMsg<>(null, msg), callback); + } + } + + public void stop() { + producer.stop(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java new file mode 100644 index 0000000000..0b0805639a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyType; + +public record DataKey(EntityKeyType type, String key, Integer keyId) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java new file mode 100644 index 0000000000..6a7c1e0ce0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +public class EdqsCountQuery extends EdqsQuery { + + @Builder + EdqsCountQuery(EntityFilter entityFilter, boolean hasKeyFilters, List keyFilters) { + super(entityFilter, hasKeyFilters, keyFilters); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java new file mode 100644 index 0000000000..7f70d2ac6c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.util.CollectionsUtil; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Getter +public class EdqsDataQuery extends EdqsQuery { + + private final int pageSize; + private final int page; + private final boolean hasTextSearch; + private final String textSearch; + private final boolean defaultSort; + private final DataKey sortKey; + private final EntityDataSortOrder.Direction sortDirection; + private final List entityFields; + private final List latestValues; + + @Builder + public EdqsDataQuery(EntityFilter entityFilter, List keyFilters, + int pageSize, int page, String textSearch, DataKey sortKey, EntityDataSortOrder.Direction sortDirection, + List entityFields, List latestValues) { + super(entityFilter, CollectionsUtil.isNotEmpty(keyFilters), keyFilters); + this.pageSize = pageSize; + this.page = page; + this.hasTextSearch = StringUtils.isNotBlank(textSearch); + this.textSearch = textSearch; + this.defaultSort = EntityKeyType.ENTITY_FIELD.equals(sortKey.type()) && "createdTime".equals(sortKey.key()) && EntityDataSortOrder.Direction.DESC.equals(sortDirection); + this.sortKey = sortKey; + this.sortDirection = sortDirection; + this.entityFields = entityFields; + this.latestValues = latestValues; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java new file mode 100644 index 0000000000..a728ea567e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; + +public record EdqsFilter(DataKey key, EntityKeyValueType valueType, KeyFilterPredicate predicate) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java new file mode 100644 index 0000000000..31e2c482b1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +@Data +public abstract class EdqsQuery { + + private final EntityFilter entityFilter; + private final boolean hasKeyFilters; + private final List keyFilters; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java new file mode 100644 index 0000000000..7185cfc63f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.edqs.data.EntityData; + +import java.util.UUID; + +@Data +public class SortableEntityData { + + private final EntityData entityData; + private String sortValue; + private boolean readAttrs; + private boolean readTs; + + public UUID getId(){ + return entityData.getId(); + } + + public EntityId getEntityId() { + return EntityIdFactory.getByTypeAndUuid(entityData.getEntityType(), entityData.getId()); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java new file mode 100644 index 0000000000..87df5586c1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public abstract class AbstractEntityGroupQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + public AbstractEntityGroupQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + var genericReadResults = processCustomerGenericRead(customerId, readAttrPermissions, readTsPermissions); + Map mergedResult = new HashMap<>(genericReadResults.size()); + for (SortableEntityData sd : genericReadResults) { + mergedResult.put(sd.getId(), sd); + } + for (GroupPermissions permissions : groupPermissions) { + SortableEntityData alreadyAdded = mergedResult.get(permissions.groupId); + if (alreadyAdded != null) { + alreadyAdded.setReadAttrs(alreadyAdded.isReadAttrs() || permissions.readAttrs); + alreadyAdded.setReadTs(alreadyAdded.isReadTs() || permissions.readTs); + } else { + EntityGroupData egData = repository.getEntityGroup(permissions.groupId); + if (matches(egData)) { + SortableEntityData sortData = toSortData(egData, permissions); + mergedResult.put(egData.getId(), sortData); + } + } + } + return new ArrayList<>(mergedResult.values()); + } + + @Override + protected CombinedPermissions getCombinedPermissionsInternal(UUID id, boolean read, boolean readAttrs, boolean readTs, List groupPermissions) { + for (GroupPermissions eg : groupPermissions) { + if (read && readAttrs && readTs) { + break; + } + if (eg.groupId.equals(id)) { + read = true; + readAttrs = readAttrs || eg.readAttrs; + readTs = readTs || eg.readTs; + } + } + return new CombinedPermissions(read, readAttrs, readTs); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java new file mode 100644 index 0000000000..e5e9dd6364 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileNames; + private final Pattern pattern; + + public AbstractEntityProfileNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + entityProfileNames = new HashSet<>(getProfileNames(this.filter)); + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && entityProfileNames.contains(ed.getFields().getType()) + && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java new file mode 100644 index 0000000000..96c3186358 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + private final Pattern pattern; + + public AbstractEntityProfileQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + var profileNamesSet = new HashSet<>(getProfileNames(this.filter)); + for (EntityData dp : repo.getEntitySet(getProfileEntityType())) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + protected abstract EntityType getProfileEntityType(); + + @Override + protected boolean matches(EntityData ed) { + ProfileAwareData profileAwareData = (ProfileAwareData) ed; + return super.matches(ed) && entityProfileIds.contains(profileAwareData.getFields().getProfileId()) + && (pattern == null || pattern.matcher(profileAwareData.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java new file mode 100644 index 0000000000..a5f7621d43 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntitySearchQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; + +public abstract class AbstractEntitySearchQueryProcessor extends AbstractRelationQueryProcessor { + + + public AbstractEntitySearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + @Override + public Set getRootEntities() { + return Set.of(filter.getRootEntity().getId()); + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + public abstract EntityType getEntityType(); + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData target = relationInfo.getTarget(); + return (filter.getRelationType() == null || relationInfo.getType().equals(filter.getRelationType())) && + getEntityType().equals(target.getEntityType()) && super.matches(target); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java new file mode 100644 index 0000000000..9efc005df2 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupTypePermissionInfo; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.checkFilters; +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public abstract class AbstractQueryProcessor implements EntityQueryProcessor { + + protected final TenantRepo repository; + protected final QueryContext ctx; + protected final EdqsQuery query; + protected final DataKey sortKey; + protected final T filter; + + public AbstractQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + this.repository = repository; + this.ctx = ctx; + this.query = query; + this.sortKey = query instanceof EdqsDataQuery dataQuery ? dataQuery.getSortKey() : null; + this.filter = filter; + } + + protected CombinedPermissions getCombinedPermissions(UUID id, boolean genericRead, boolean genericAttrs, boolean genericTs, List groupPermissions) { + return getCombinedPermissionsInternal(id, genericRead, genericRead && genericAttrs, genericRead && genericTs, groupPermissions); + } + + protected CombinedPermissions getCombinedPermissions(UUID id, List groupPermissions) { + return getCombinedPermissionsInternal(id, false, false, false, groupPermissions); + } + + protected CombinedPermissions getCombinedPermissionsInternal(UUID id, boolean read, boolean readAttrs, boolean readTs, List groupPermissions) { + for (GroupPermissions eg : groupPermissions) { + if (read && readAttrs && readTs) { + break; + } + boolean hasMorePermissions = !read || (!readAttrs && eg.readAttrs) || (!readTs && eg.readTs); + if (hasMorePermissions && repository.contains(eg.groupId, id)) { + read = true; + readAttrs = readAttrs || eg.readAttrs; + readTs = readTs || eg.readTs; + } + } + return new CombinedPermissions(read, readAttrs, readTs); + } + + protected SortableEntityData toSortDataGroupsOnly(EntityData ed, List groupPermissions) { + SortableEntityData sortData; + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), groupPermissions); + if (permissions.isRead()) { + sortData = toSortData(ed, permissions); + } else { + sortData = null; + } + return sortData; + } + + protected SortableEntityData toSortData(EntityData ed, boolean readAttrs, boolean readTs) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(readAttrs); + sortData.setReadTs(readTs); + return sortData; + } + + protected SortableEntityData toSortData(EntityData ed, Permissions permissions) { + return toSortData(ed, permissions.isReadAttrs(), permissions.isReadTs()); + } + + protected static List toGroupPermissions(MergedGroupTypePermissionInfo readPermissions, + MergedGroupTypePermissionInfo readAttrPermissions, + MergedGroupTypePermissionInfo readTsPermissions) { + List permissions = new ArrayList<>(); + for (EntityGroupId egId : readPermissions.getEntityGroupIds()) { + permissions.add(new GroupPermissions(egId.getId(), + readAttrPermissions.getEntityGroupIds() != null && readAttrPermissions.getEntityGroupIds().contains(egId), + readTsPermissions.getEntityGroupIds() != null && readTsPermissions.getEntityGroupIds().contains(egId))); + } + return permissions; + } + + protected static boolean checkCustomerHierarchy(Set customers, EntityData ed) { + return ed.getCustomerId() != null && customers.contains(ed.getCustomerId()); + } + + protected void process(Collection> entities, Consumer> processor) { + for (EntityData ed : entities) { + if (matches(ed)) { + processor.accept(ed); + } + } + } + + protected boolean matches(EntityData ed) { + return checkFilters(query, ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java new file mode 100644 index 0000000000..755c79cea6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -0,0 +1,260 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public abstract class AbstractRelationQueryProcessor extends AbstractQueryProcessor { + + public static final int MAXIMUM_QUERY_LEVEL = 100; + + public AbstractRelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + protected abstract Set getRootEntities(); + + protected abstract EntitySearchDirection getDirection(); + + protected abstract int getMaxLevel(); + + protected abstract boolean isFetchLastLevelOnly(); + + protected boolean isMultiRoot() { + return false; + } + + @Override + public List processQuery() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + if (ctx.isTenantUser()) { + return processTenantQuery(entities); + } else { + return processCustomerQuery(entities); + } + } + + @Override + public long count() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + long result = 0; + + RelationQueryPermissions[] permissionsArray = buildPermissionsArray(); + if (ctx.isTenantUser()) { + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + if (permissions.isHasGroups()) { + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + permissions.isReadEntity(), permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + result++; + } + } else if (permissions.isReadEntity()) { + result++; + } + } + } + } else { + var customerIds = repository.getAllCustomers(ctx.getCustomerId().getId()); + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + boolean isReadEntity = permissions.isReadEntity() && ed.getCustomerId() != null && customerIds.contains(ed.getCustomerId()); + if (permissions.isHasGroups()) { + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + isReadEntity, + permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + result++; + } + } else if (isReadEntity) { + result++; + } + } + } + return result; + } + return result; + } + + private List processTenantQuery(Set> entities) { + List result = new ArrayList<>(); + RelationQueryPermissions[] permissionsArray = buildPermissionsArray(); + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + if (permissions.isHasGroups()) { + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + permissions.isReadEntity(), permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(combinedPermissions.isReadAttrs()); + sortData.setReadTs(combinedPermissions.isReadTs()); + result.add(sortData); + } + } else if (permissions.isReadEntity()) { + result.add(toSortData(ed, permissions)); + } + } + } + return result; + } + + private List processCustomerQuery(Set> entities) { + var customerIds = repository.getAllCustomers(ctx.getCustomerId().getId()); + RelationQueryPermissions[] permissionsArray = buildPermissionsArray(); + List result = new ArrayList<>(); + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + boolean isReadEntity = permissions.isReadEntity() && ed.getCustomerId() != null && customerIds.contains(ed.getCustomerId()); + if (permissions.isHasGroups()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + isReadEntity, + permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + sortData.setReadAttrs(combinedPermissions.isReadAttrs()); + sortData.setReadTs(combinedPermissions.isReadTs()); + result.add(sortData); + } + } else if (isReadEntity) { + result.add(toSortData(ed, permissions)); + } + } + } + return result; + } + + private RelationQueryPermissions[] buildPermissionsArray() { + RelationQueryPermissions[] permissionsArray = new RelationQueryPermissions[EntityType.values().length]; + var readEntityPermissionsMap = ctx.getMergedReadEntityPermissionsMap(); + var readAttrPermissionsMap = ctx.getMergedReadAttrPermissionsMap(); + var readTsPermissionsMap = ctx.getMergedReadTsPermissionsMap(); + for (EntityType et : EntityType.values()) { + var resource = Resource.resourceFromEntityType(et); + if (resource == null) { + continue; + } + var readEntityPermissions = readEntityPermissionsMap.get(resource); + var readAttrPermissions = readAttrPermissionsMap.get(resource); + var readTsPermissions = readTsPermissionsMap.get(resource); + var groupPermissions = toGroupPermissions(readEntityPermissions, readAttrPermissions, readTsPermissions); + RelationQueryPermissions entityPermissions = RelationQueryPermissions + .builder() + .readEntity(readEntityPermissions.isHasGenericRead()) + .readAttrs(readAttrPermissions.isHasGenericRead()) + .readTs(readTsPermissions.isHasGenericRead()) + .hasGroups(!groupPermissions.isEmpty()) + .groupPermissions(groupPermissions) + .build(); + permissionsArray[et.ordinal()] = entityPermissions; + } + return permissionsArray; + } + + private Set> getEntitiesSet(RelationsRepo relations) { + Set> result = new HashSet<>(); + Set processed = new HashSet<>(); + Queue tasks = new LinkedList<>(); + int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel()); + for (UUID uuid : getRootEntities()) { + tasks.add(new RelationSearchTask(uuid, 0)); + } + while (!tasks.isEmpty()) { + RelationSearchTask task = tasks.poll(); + if (processed.add(task.entityId)) { + var entityLvl = task.lvl + 1; + Set entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId); + if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) { + result.add(task.previous.getTarget()); + } + for (RelationInfo relationInfo : entities) { + var entity = relationInfo.getTarget(); + if (entity.isEmpty()) { + continue; + } + var entityId = entity.getId(); + if (isFetchLastLevelOnly()) { + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl, relationInfo)); + } else if (entityLvl == maxLvl) { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + } + } else { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl)); + } + } + } + } + } + return result; + } + + protected abstract boolean check(RelationInfo relationInfo); + + @RequiredArgsConstructor + private static class RelationSearchTask { + private final UUID entityId; + private final int lvl; + private final RelationInfo previous; + + public RelationSearchTask(UUID entityId, int lvl) { + this(entityId, lvl, null); + } + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java new file mode 100644 index 0000000000..2a9ef090e7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; + +public abstract class AbstractSimpleQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + + public AbstractSimpleQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter); + this.entityType = entityType; + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getEntityMap(EntityType.CUSTOMER); + for (UUID cId : repository.getAllCustomers(customerId)) { + var customerData = (CustomerData) customers.get(cId); + if (customerData != null) { + process(customerData.getEntities(entityType), processor); + } + } + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + var genericReadResults = processCustomerGenericRead(customerId, readAttrPermissions, readTsPermissions); + Map mergedResult = new HashMap<>(genericReadResults.size()); + for (SortableEntityData sd : genericReadResults) { + mergedResult.put(sd.getId(), sd); + } + + for (GroupPermissions permissions : groupPermissions) { + EntityGroupData egData = repository.getEntityGroup(permissions.groupId); + for (EntityData ed : egData.getEntities()) { + SortableEntityData alreadyAdded = mergedResult.get(ed.getId()); + if (alreadyAdded != null) { + alreadyAdded.setReadAttrs(alreadyAdded.isReadAttrs() || permissions.readAttrs); + alreadyAdded.setReadTs(alreadyAdded.isReadTs() || permissions.readTs); + } else { + if (matches(ed)) { + SortableEntityData sortData = toSortData(ed, permissions); + mergedResult.put(ed.getId(), sortData); + } + } + } + } + return new ArrayList<>(mergedResult.values()); + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + for (GroupPermissions groupPermission : groupPermissions) { + EntityGroupData egData = repository.getEntityGroup(groupPermission.groupId); + process(egData.getEntities(), processor); + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(entityType), processor); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java new file mode 100644 index 0000000000..9411d75935 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java @@ -0,0 +1,172 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +public abstract class AbstractSingleEntityTypeQueryProcessor extends AbstractQueryProcessor { + + public AbstractSingleEntityTypeQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + super(repository, ctx, query, filter); + } + + @Override + public List processQuery() { + var readPermissions = ctx.getMergedReadPermissionsByEntityType(); + if (readPermissions == null) { + return Collections.emptyList(); + } + var readAttrPermissions = ctx.getMergedReadAttrPermissionsByEntityType(); + var readTsPermissions = ctx.getMergedReadTsPermissionsByEntityType(); + + boolean hasGenericRead = readPermissions.isHasGenericRead(); + boolean hasGroups = readPermissions.getEntityGroupIds() != null && !readPermissions.getEntityGroupIds().isEmpty(); + if (!hasGenericRead && !hasGroups) { + return Collections.emptyList(); + } + boolean hasGenericAttrRead = readAttrPermissions.isHasGenericRead(); + boolean hasGenericTsRead = readTsPermissions.isHasGenericRead(); + + if (hasGenericRead) { + if (ctx.isTenantUser()) { + if (hasGroups && (!hasGenericAttrRead || !hasGenericTsRead)) { + return processTenantGenericReadWithGroups(hasGenericAttrRead, hasGenericTsRead, + toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)); + } else { + return processTenantGenericRead(hasGenericAttrRead, hasGenericTsRead); + } + } else { + if (hasGroups) { + return processCustomerGenericReadWithGroups(ctx.getCustomerId().getId(), hasGenericAttrRead, hasGenericTsRead, + toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)); + } else { + return processCustomerGenericRead(ctx.getCustomerId().getId(), hasGenericAttrRead, hasGenericTsRead); + } + } + } else { + return processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)); + } + } + + @Override + public long count() { // TODO: get rid of the duplicates + var readPermissions = ctx.getMergedReadPermissionsByEntityType(); + if (readPermissions == null) { + return 0; + } + var readAttrPermissions = ctx.getMergedReadAttrPermissionsByEntityType(); + var readTsPermissions = ctx.getMergedReadTsPermissionsByEntityType(); + boolean hasGenericRead = readPermissions.isHasGenericRead(); + boolean hasGroups = readPermissions.getEntityGroupIds() != null && !readPermissions.getEntityGroupIds().isEmpty(); + + if (!hasGenericRead && !hasGroups && !ctx.isIgnorePermissionCheck()) { + return 0; + } + + AtomicLong result = new AtomicLong(); + Consumer> counter = ed -> result.incrementAndGet(); + + if (ctx.isIgnorePermissionCheck()) { + processAll(counter); + } else if (ctx.isTenantUser()) { + if (hasGenericRead) { + processAll(counter); + } else { + processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions), counter); + } + } else { + if (hasGenericRead) { + if (hasGroups) { + result.addAndGet(processCustomerGenericReadWithGroups(ctx.getCustomerId().getId(), readAttrPermissions.isHasGenericRead(), readTsPermissions.isHasGenericRead(), + toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)).size()); // FIXME: not efficient + } else { + processCustomerGenericRead(ctx.getCustomerId().getId(), counter); + } + } else { + processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions), counter); + } + } + return result.get(); + } + + protected List processTenantGenericRead(boolean readAttrPermissions, + boolean readTsPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processAll(ed -> { + result.add(toSortData(ed, readAttrPermissions, readTsPermissions)); + }); + return result; + } + + protected List processCustomerGenericRead(UUID customerId, + boolean readAttrPermissions, + boolean readTsPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processCustomerGenericRead(customerId, ed -> { + result.add(toSortData(ed, readAttrPermissions, readTsPermissions)); + }); + return result; + } + + protected abstract void processCustomerGenericRead(UUID customerId, Consumer> processor); + + protected List processTenantGenericReadWithGroups(boolean readAttrPermissions, + boolean readTsPermissions, + List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), true, readAttrPermissions, readTsPermissions, groupPermissions); + SortableEntityData sortData = toSortData(ed, permissions); + result.add(sortData); + }); + return result; + } + + protected abstract List processCustomerGenericReadWithGroups(UUID customerId, + boolean readAttrPermissions, + boolean readTsPermissions, + List groupPermissions); + + protected List processGroupsOnly(List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processGroupsOnly(groupPermissions, ed -> { + SortableEntityData sortData = toSortDataGroupsOnly(ed, groupPermissions); + if (sortData != null) { + result.add(sortData); + } + }); + return result; + } + + protected abstract void processGroupsOnly(List groupPermissions, Consumer> processor); + + protected abstract void processAll(Consumer> processor); + + protected abstract int getProbableResultSize(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java new file mode 100644 index 0000000000..c2821161c7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +public class ApiUsageStateQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + public ApiUsageStateQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (ApiUsageStateFilter) query.getEntityFilter()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + process(customerData.getEntities(EntityType.API_USAGE_STATE), processor); + } + + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + Collection> entities = customerData.getEntities(EntityType.API_USAGE_STATE); + EntityData ed = entities.iterator().next(); + if (entities.isEmpty() || !matches(ed)) { + return Collections.emptyList(); + } else { + boolean genericRead = customerId.equals(ed.getCustomerId()); + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = toSortData(customerData, permissions); + return Collections.singletonList(sortData); + } else { + return Collections.emptyList(); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(EntityType.API_USAGE_STATE), processor); + } + + @Override + protected boolean matches(EntityData ed) { + ApiUsageStateFields entityFields = (ApiUsageStateFields) ed.getFields(); + return super.matches(ed) && (filter.getCustomerId() != null ? entityFields.getEntityId().equals(filter.getCustomerId()) : + entityFields.getEntityId().equals(repository.getTenantId())); + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java new file mode 100644 index 0000000000..89826fe1be --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class AssetSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public AssetSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getAssetTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getAssetTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.ASSET_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java new file mode 100644 index 0000000000..d012e12e46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class AssetTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public AssetTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetTypeFilter) query.getEntityFilter(), EntityType.ASSET); + } + + @Override + protected String getEntityNameFilter(AssetTypeFilter filter) { + return filter.getAssetNameFilter(); + } + + @Override + protected List getProfileNames(AssetTypeFilter filter) { + return filter.getAssetTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.ASSET_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java new file mode 100644 index 0000000000..3ff6879d46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.Data; + +@Data +public class CombinedPermissions implements Permissions { + private final boolean read; + private final boolean readAttrs; + private final boolean readTs; +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java new file mode 100644 index 0000000000..e76f0159bf --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class DeviceSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public DeviceSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getDeviceTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getDeviceTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.DEVICE_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java new file mode 100644 index 0000000000..19068eb160 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class DeviceTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public DeviceTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceTypeFilter) query.getEntityFilter(), EntityType.DEVICE); + } + + @Override + protected String getEntityNameFilter(DeviceTypeFilter filter) { + return filter.getDeviceNameFilter(); + } + + @Override + protected List getProfileNames(DeviceTypeFilter filter) { + return filter.getDeviceTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java new file mode 100644 index 0000000000..2c5c7b1dd2 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EdgeTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EdgeTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeTypeFilter) query.getEntityFilter(), EntityType.EDGE); + } + + @Override + protected String getEntityNameFilter(EdgeTypeFilter filter) { + return filter.getEdgeNameFilter(); + } + + @Override + protected List getProfileNames(EdgeTypeFilter filter) { + return filter.getEdgeTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java new file mode 100644 index 0000000000..3ea4c247a9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EdgeTypeSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EdgeTypeSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEdgeTypes() == null || filter.getEdgeTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java new file mode 100644 index 0000000000..cc3c26dc18 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java @@ -0,0 +1,121 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntitiesByGroupNameFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.regex.Pattern; + +import static org.thingsboard.server.common.data.EntityType.ENTITY_GROUP; +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public class EntitiesByGroupNameQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final String groupType; + private final UUID ownerId; + private final Pattern pattern; + + public EntitiesByGroupNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntitiesByGroupNameFilter) query.getEntityFilter()); + this.groupType = filter.getGroupType().name(); + this.ownerId = filter.getOwnerId() != null ? filter.getOwnerId().getId() : null; + this.pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityGroupNameFilter()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getEntityMap(EntityType.CUSTOMER); + for (UUID cId : repository.getAllCustomers(customerId)) { + var customerData = (CustomerData) customers.get(cId); + if (customerData != null) { + process(customerData.getEntities(ENTITY_GROUP), processor); + } + } + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(permissions.isReadAttrs()); + sortData.setReadTs(permissions.isReadTs()); + result.add(sortData); + } + }); + return result; + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + Collection> entities = new HashSet<>(getProbableResultSize()); + for (GroupPermissions groupPermission : groupPermissions) { + entities.add(repository.getEntityGroup(groupPermission.groupId)); + } + process(entities, processor); + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(ENTITY_GROUP), processor); + } + + @Override + protected void process(Collection> entities, Consumer> processor) { + for (EntityData ed : entities) { + if (matches(ed)) { + Collection> groupEntities = repository.getEntityGroup(ed.getId()).getEntities(); + for (EntityData groupEntity : groupEntities) { + processor.accept(groupEntity); + } + return; + } + } + } + + @Override + protected boolean matches(EntityData ed) { + EntityGroupFields fields = (EntityGroupFields)ed.getFields(); + return super.matches(ed) && groupType.equals(fields.getType()) + && (pattern == null || pattern.matcher(fields.getName()).matches()) + && (ownerId == null || ownerId.equals(fields.getOwnerId())); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java new file mode 100644 index 0000000000..feb51e5f46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java @@ -0,0 +1,96 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityGroupFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public class EntitiesByGroupQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final String groupType; + private final UUID groupId; + + public EntitiesByGroupQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityGroupFilter) query.getEntityFilter()); + groupId = UUID.fromString(filter.getEntityGroup()); + groupType = filter.getGroupType().name(); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + if (checkCustomerHierarchy(customers, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(permissions.isReadAttrs()); + sortData.setReadTs(permissions.isReadTs()); + result.add(sortData); + } + }); + return result; + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + EntityGroupData entityGroup = repository.getEntityGroup(groupId); + if (matches(entityGroup)) { + for (EntityData ed : entityGroup.getEntities()) { + processor.accept(ed); + } + } + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && groupType.equals(ed.getFields().getType()); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java new file mode 100644 index 0000000000..7022ca5cb6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityGroupListFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +public class EntityGroupListQueryProcessor extends AbstractEntityGroupQueryProcessor { + + private final String groupType; + private final Set groupIds; + + public EntityGroupListQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityGroupListFilter) query.getEntityFilter()); + this.groupType = filter.getGroupType().name(); + this.groupIds = filter.getEntityGroupList().stream().map(UUID::fromString).collect(Collectors.toSet()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + if (checkCustomerHierarchy(customers, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + Set allowedGroupIds = groupPermissions.stream().map(GroupPermissions::getGroupId) + .filter(this.groupIds::contains).collect(Collectors.toSet()); + + checkGroupIds(allowedGroupIds, processor); + } + + @Override + protected void processAll(Consumer> processor) { + checkGroupIds(groupIds, processor); + } + + @Override + protected int getProbableResultSize() { + return groupIds.size(); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && groupType.equals(ed.getFields().getType()); + } + + private void checkGroupIds(Set allowedGroupIds, Consumer> processor) { + for (UUID groupId : allowedGroupIds) { + EntityGroupData entityGroup = repository.getEntityGroup(groupId); + if (matches(entityGroup)) { + processor.accept(entityGroup); + } + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java new file mode 100644 index 0000000000..9491142619 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityGroupNameFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.regex.Pattern; + +import static org.thingsboard.server.common.data.EntityType.ENTITY_GROUP; + +public class EntityGroupNameQueryProcessor extends AbstractEntityGroupQueryProcessor { + + private final String groupType; + private final Pattern groupNamePattern; + + public EntityGroupNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityGroupNameFilter) query.getEntityFilter()); + this.groupType = filter.getGroupType().name(); + this.groupNamePattern = RepositoryUtils.toSqlLikePattern(filter.getEntityGroupNameFilter()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getEntityMap(EntityType.CUSTOMER); + for (UUID cId : repository.getAllCustomers(customerId)) { + var customerData = (CustomerData) customers.get(cId); + if (customerData != null) { + process(customerData.getEntities(ENTITY_GROUP), processor); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + for (GroupPermissions groupPermission : groupPermissions) { + EntityGroupData entityGroup = repository.getEntityGroup(groupPermission.groupId); + if (matches(entityGroup)) { + processor.accept(entityGroup); + } + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(ENTITY_GROUP), processor); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && (groupNamePattern == null || groupNamePattern.matcher(ed.getFields().getName()).matches()) + && groupType.equals(ed.getFields().getType()); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java new file mode 100644 index 0000000000..495f925bd0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final Set entityIds; + + public EntityListQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityListFilter) query.getEntityFilter()); + this.entityType = filter.getEntityType(); + this.entityIds = filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toSet()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + if (checkCustomerHierarchy(customers, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(permissions.isReadAttrs()); + sortData.setReadTs(permissions.isReadTs()); + result.add(sortData); + } + }); + return result; + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + var map = repository.getEntityMap(entityType); + for (UUID entityId : entityIds) { + EntityData ed = map.get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + } + + @Override + protected int getProbableResultSize() { + return entityIds.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java new file mode 100644 index 0000000000..3b64706bfe --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.regex.Pattern; + +public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Pattern pattern; + + public EntityNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityNameFilter) query.getEntityFilter(), ((EntityNameFilter) query.getEntityFilter()).getEntityType()); + pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityNameFilter()); + } + + @Override + protected boolean matches(EntityData ed) { + return ed.getFields() != null && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java new file mode 100644 index 0000000000..a42c79b61f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.edqs.query.SortableEntityData; + +import java.util.List; + +public interface EntityQueryProcessor { + + List processQuery(); + + long count(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java new file mode 100644 index 0000000000..271ff01425 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityQueryProcessorFactory { + + public static EntityQueryProcessor create(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + return switch (query.getEntityFilter().getType()) { + case SINGLE_ENTITY -> new SingleEntityQueryProcessor(repo, ctx, query); + case ENTITY_LIST -> new EntityListQueryProcessor(repo, ctx, query); + case ENTITY_NAME -> new EntityNameQueryProcessor(repo, ctx, query); + case ENTITY_TYPE -> new EntityTypeQueryProcessor(repo, ctx, query); + case DEVICE_TYPE -> new DeviceTypeQueryProcessor(repo, ctx, query); + case ASSET_TYPE -> new AssetTypeQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_TYPE -> new EntityViewTypeQueryProcessor(repo, ctx, query); + case EDGE_TYPE -> new EdgeTypeQueryProcessor(repo, ctx, query); + case RELATIONS_QUERY -> new RelationQueryProcessor(repo, ctx, query); + case ENTITY_GROUP -> new EntitiesByGroupQueryProcessor(repo, ctx, query); + case ENTITY_GROUP_LIST -> new EntityGroupListQueryProcessor(repo, ctx, query); + case ENTITY_GROUP_NAME -> new EntityGroupNameQueryProcessor(repo, ctx, query); + case ENTITIES_BY_GROUP_NAME -> new EntitiesByGroupNameQueryProcessor(repo, ctx, query); + case STATE_ENTITY_OWNER -> new StateEntityOwnerQueryProcessor(repo, ctx, query); + case API_USAGE_STATE -> new ApiUsageStateQueryProcessor(repo, ctx, query); + case ASSET_SEARCH_QUERY -> new AssetSearchQueryProcessor(repo, ctx, query); + case DEVICE_SEARCH_QUERY -> new DeviceSearchQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_SEARCH_QUERY -> new EntityViewSearchQueryProcessor(repo, ctx, query); + case EDGE_SEARCH_QUERY -> new EdgeTypeSearchQueryProcessor(repo, ctx, query); + case SCHEDULER_EVENT -> new SchedulerEventQueryProcessor(repo, ctx, query); + default -> throw new RuntimeException("Not Implemented!"); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java new file mode 100644 index 0000000000..b390a3c80c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityTypeQueryProcessor extends AbstractSimpleQueryProcessor { + + public EntityTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityTypeFilter) query.getEntityFilter(), ((EntityTypeFilter) query.getEntityFilter()).getEntityType()); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java new file mode 100644 index 0000000000..f16de48b6a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityViewSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EntityViewSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEntityViewTypes() == null || filter.getEntityViewTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java new file mode 100644 index 0000000000..3932cd15fd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EntityViewTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EntityViewTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewTypeFilter) query.getEntityFilter(), EntityType.ENTITY_VIEW); + } + + @Override + protected String getEntityNameFilter(EntityViewTypeFilter filter) { + return filter.getEntityViewNameFilter(); + } + + @Override + protected List getProfileNames(EntityViewTypeFilter filter) { + return filter.getEntityViewTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java new file mode 100644 index 0000000000..65a5a7d9b0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.Data; + +import java.util.UUID; + +@Data +public class GroupPermissions implements Permissions { + + protected final UUID groupId; + protected final boolean readAttrs; + protected final boolean readTs; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java new file mode 100644 index 0000000000..e2231a8df2 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +public interface Permissions { + + boolean isReadAttrs(); + + boolean isReadTs(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java new file mode 100644 index 0000000000..fb151d85af --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.Builder; +import lombok.Getter; + +import java.util.List; + +@Getter +@Builder +public class RelationQueryPermissions implements Permissions { + + private final boolean readEntity; + private final boolean readAttrs; + private final boolean readTs; + private final boolean hasGroups; + private final List groupPermissions; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java new file mode 100644 index 0000000000..8f70344239 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class RelationQueryProcessor extends AbstractRelationQueryProcessor { + + private final boolean hasFilters; + + public RelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (RelationsQueryFilter) query.getEntityFilter()); + this.hasFilters = filter.getFilters() != null && !filter.getFilters().isEmpty(); + } + + @Override + public Set getRootEntities() { + if (filter.isMultiRoot()) { + return filter.getMultiRootEntityIds().stream().map(UUID::fromString).collect(Collectors.toSet()); + } else { + return Set.of(filter.getRootEntity().getId()); + } + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isMultiRoot() { + return filter.isMultiRoot(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + @Override + protected boolean check(RelationInfo relationInfo) { + if (hasFilters) { + for (var f : filter.getFilters()) { + if (((!filter.isNegate() && !f.isNegate()) || (filter.isNegate() && f.isNegate())) == f.getRelationType().equals(relationInfo.getType())) { + if (f.getEntityTypes() == null || f.getEntityTypes().isEmpty() + || f.getEntityTypes().contains(relationInfo.getTarget().getEntityType())) { + return super.matches(relationInfo.getTarget()); + } + } + } + return false; + } else { + return super.matches(relationInfo.getTarget()); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java new file mode 100644 index 0000000000..3c48b89cb5 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.SchedulerEventFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class SchedulerEventQueryProcessor extends AbstractSimpleQueryProcessor { + + public SchedulerEventQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (SchedulerEventFilter) query.getEntityFilter(), EntityType.SCHEDULER_EVENT); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && (filter.getEventType() == null || filter.getEventType().equals(ed.getFields().getType())) + && (filter.getOriginator() == null || filter.getOriginator().equals(ed.getFields().getOriginatorId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java new file mode 100644 index 0000000000..ee3810ca6b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +public class SingleEntityQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final UUID entityId; + + public SingleEntityQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (SingleEntityFilter) query.getEntityFilter()); + this.entityType = filter.getSingleEntity().getEntityType(); + this.entityId = filter.getSingleEntity().getId(); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (ed != null && ed.getCustomerId() != null && matches(ed)) { + if (customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId())) { + processor.accept(ed); + } + } + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (!matches(ed)) { + return Collections.emptyList(); + } else { + boolean genericRead = customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId()); + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = toSortData(ed, permissions); + return Collections.singletonList(sortData); + } else { + return Collections.emptyList(); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java new file mode 100644 index 0000000000..0e79f21747 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +public class StateEntityOwnerQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityId entityId; + + public StateEntityOwnerQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (StateEntityOwnerFilter) query.getEntityFilter()); + this.entityId = filter.getSingleEntity(); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId()); + if (ed != null && ed.getCustomerId() != null && matches(ed)) { + if (customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId())) { + processor.accept(ed); + } + } + } + + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId()); + if (!matches(ed)) { + return Collections.emptyList(); + } else { + boolean genericRead = customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId()); + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = toSortData(ed, permissions); + return Collections.singletonList(sortData); + } else { + return Collections.emptyList(); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId()); + if (ed != null) { + if (ed.getCustomerId() != null) { + processor.accept(repository.getEntityMap(EntityType.CUSTOMER).get(ed.getCustomerId())); + } else { + processor.accept(repository.getEntityMap(EntityType.TENANT).get(repository.getTenantId().getId())); + } + } + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java new file mode 100644 index 0000000000..9e829dfac1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +import java.util.function.Predicate; + +public interface EdqRepository { + + void processEvent(EdqsEvent event); + + @Deprecated + default void addOrUpdate(TenantId tenantId, Object object) {} + + long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery query, boolean ignorePermissionCheck); + + PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityDataQuery query, boolean ignorePermissionCheck); + + void clearIf(Predicate predicate); + + void clear(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java new file mode 100644 index 0000000000..2934fdbfba --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Predicate; + +@EdqsComponent +@AllArgsConstructor +@Service +@Slf4j +public class InMemoryEdqRepository implements EdqRepository { + + private final static ConcurrentMap repos = new ConcurrentHashMap<>(); + private final Optional statsService; + + public TenantRepo get(TenantId tenantId) { + return repos.computeIfAbsent(tenantId, id -> new TenantRepo(id, statsService)); + } + + @Override + public void processEvent(EdqsEvent event) { + get(event.getTenantId()).processEvent(event); + } + + @Override + public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + long result = 0; + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + for (TenantRepo repo : repos.values()) { + result += repo.countEntitiesByQuery(customerId, userPermissions, query, ignorePermissionCheck); + } + } else { + result = get(tenantId).countEntitiesByQuery(customerId, userPermissions, query, ignorePermissionCheck); + } + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("countEntitiesByQuery: {} ms", timingMs); + return result; + } + + @Override + public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, + MergedUserPermissions userPermissions, EntityDataQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + var result = get(tenantId).findEntityDataByQuery(customerId, userPermissions, query, ignorePermissionCheck); + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("findEntityDataByQuery: {} ms", timingMs); + return result; + } + + @Override + public void clearIf(Predicate predicate) { + repos.keySet().removeIf(predicate); + } + + @Override + public void clear() { + repos.clear(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java new file mode 100644 index 0000000000..f2dcd41e7c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class KeyDictionary { + + private static final ConcurrentMap keyToIdDict = new ConcurrentHashMap<>(); + private static final ConcurrentMap idToKeyDict = new ConcurrentHashMap<>(); + private static final AtomicInteger keySeq = new AtomicInteger(); + + public static Integer get(String key) { + return keyToIdDict.computeIfAbsent(key, __ -> { + int keyId = keySeq.incrementAndGet(); + idToKeyDict.put(keyId, key); + return keyId; + }); + } + + public static String get(Integer keyId) { + return idToKeyDict.get(keyId); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java new file mode 100644 index 0000000000..876d92ee82 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import com.google.common.hash.Hashing; +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbBytePool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static byte[] intern(byte[] data) { + if (data == null) { + return null; + } + var checksum = Hashing.sha512().hashBytes(data).toString(); + return pool.computeIfAbsent(checksum, c -> data); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java new file mode 100644 index 0000000000..177651248b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbStringPool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static String intern(String data) { + if (data == null) { + return null; + } + return pool.computeIfAbsent(data, str -> str); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java new file mode 100644 index 0000000000..93422eb5fd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -0,0 +1,584 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.ApiUsageStateData; +import org.thingsboard.server.edqs.data.AssetData; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.GenericData; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.data.TenantData; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedStringDataPoint; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.JsonDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessor; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessorFactory; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_ASC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_DESC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.SYS_ADMIN_PERMISSIONS; +import static org.thingsboard.server.edqs.util.RepositoryUtils.resolveEntityType; + +@Slf4j +public class TenantRepo { + + public static final Comparator> CREATED_TIME_COMPARATOR = Comparator.comparingLong(o -> o.getFields() != null ? o.getFields().getCreatedTime() : 0); // FIXME: fields may be null at first + public static final Comparator> CREATED_TIME_AND_ID_COMPARATOR = CREATED_TIME_COMPARATOR + .thenComparing(EntityData::getId); + public static final Comparator> CREATED_TIME_AND_ID_DESC_COMPARATOR = CREATED_TIME_AND_ID_COMPARATOR.reversed(); + + private final ConcurrentMap>> entitySetByType = new ConcurrentHashMap<>(); + private final ConcurrentMap>> entityMapByType = new ConcurrentHashMap<>(); + private final ConcurrentMap> customersHierarchy = new ConcurrentHashMap<>(); + private final ConcurrentMap entityGroups = new ConcurrentHashMap<>(); + private final ConcurrentMap relations = new ConcurrentHashMap<>(); + + private final Lock entityUpdateLock = new ReentrantLock(); + + private final TenantId tenantId; + private final Optional edqsStatsService; + + public TenantRepo(TenantId tenantId, Optional edqsStatsService) { + this.tenantId = tenantId; + this.edqsStatsService = edqsStatsService; + } + + public void processEvent(EdqsEvent event) { + EdqsObject edqsObject = event.getObject(); + log.debug("[{}] Processing event: {}", tenantId, event); + if (event.getEventType() == EdqsEventType.UPDATED) { + addOrUpdate(edqsObject); + } else if (event.getEventType() == EdqsEventType.DELETED) { + remove(edqsObject); + } + } + + public void addOrUpdate(EdqsObject object) { + if (object instanceof EntityRelation relation) { + addOrUpdateRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + addOrUpdateAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + addOrUpdateLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + addOrUpdateEntity(entity); + } + } + + public void remove(EdqsObject object) { + if (object instanceof EntityRelation relation) { + removeRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + removeAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + removeLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + removeEntity(entity); + } + } + + private void addOrUpdateRelation(EntityRelation entity) { + entityUpdateLock.lock(); + try { + if (RelationTypeGroup.COMMON.equals(entity.getTypeGroup())) { + RelationsRepo repo = relations.computeIfAbsent(entity.getTypeGroup(), tg -> new RelationsRepo()); + EntityData from = getOrCreate(entity.getFrom()); + EntityData to = getOrCreate(entity.getTo()); + boolean added = repo.add(from, to, TbStringPool.intern(entity.getType())); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); + } + } else if (RelationTypeGroup.FROM_ENTITY_GROUP.equals(entity.getTypeGroup())) { + var eg = getEntityGroup(entity.getFrom().getId()); + if (eg != null) { + eg.addOrUpdate(getOrCreate(entity.getTo())); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + private void removeRelation(EntityRelation entityRelation) { + if (RelationTypeGroup.COMMON.equals(entityRelation.getTypeGroup())) { + RelationsRepo relationsRepo = relations.get(entityRelation.getTypeGroup()); + if (relationsRepo != null) { + boolean removed = relationsRepo.remove(entityRelation.getFrom().getId(), entityRelation.getTo().getId(), entityRelation.getType()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); + } + } + } else if (RelationTypeGroup.FROM_ENTITY_GROUP.equals(entityRelation.getTypeGroup())) { + var eg = getEntityGroup(entityRelation.getFrom().getId()); + if (eg != null) { + eg.remove(entityRelation.getTo().getId()); + } + } + } + + private void addOrUpdateEntity(Entity entity) { + entityUpdateLock.lock(); + try { + log.trace("[{}] addOrUpdateEntity: {}", tenantId, entity); + EntityFields fields = entity.getFields(); + UUID entityId = fields.getId(); + EntityType entityType = entity.getType(); + + EntityData entityData = getOrCreate(entityType, entityId); + processFields(fields); + entityData.setFields(entity.getFields()); + + switch (entity.getType()) { + case ENTITY_GROUP -> { + EntityGroupFields entityGroupFields = (EntityGroupFields) fields; + UUID ownerId = entityGroupFields.getOwnerId(); + if (EntityType.CUSTOMER.equals(entityGroupFields.getOwnerType())) { + entityData.setCustomerId(ownerId); + ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(ownerId, CustomerData::new)).addOrUpdate(entityData); + } + entityGroups.put(fields.getId(), (EntityGroupData) entityData); + } + case CUSTOMER -> { + CustomerFields customerFields = (CustomerFields) fields; + UUID newParentId = customerFields.getCustomerId(); // for customer, customerId is parentCustomerId + UUID oldParentId = entityData.getCustomerId(); + entityData.setCustomerId(newParentId); + if (entityIdMismatch(oldParentId, newParentId)) { + if (oldParentId != null) { + customersHierarchy.computeIfAbsent(oldParentId, id -> new HashSet<>()).remove(entityData.getId()); + } + if (newParentId != null) { + customersHierarchy.computeIfAbsent(newParentId, id -> new HashSet<>()).add(entityData.getId()); + } + } + } + default -> { + UUID newCustomerId = fields.getCustomerId(); + UUID oldCustomerId = entityData.getCustomerId(); + entityData.setCustomerId(newCustomerId); + if (entityIdMismatch(oldCustomerId, newCustomerId)) { + if (oldCustomerId != null) { + CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); + if (old != null) { + old.remove(entityData); + } + } + if (newCustomerId != null) { + CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); + newData.addOrUpdate(entityData); + } + } + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void removeEntity(Entity entity) { + entityUpdateLock.lock(); + try { + UUID entityId = entity.getFields().getId(); + EntityType entityType = entity.getType(); + EntityData removed = getEntityMap(entityType).remove(entityId); + if (removed != null) { + getEntitySet(entityType).remove(removed); + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + } + switch (entityType) { + case ENTITY_GROUP -> { + entityGroups.remove(entityId); + } + case CUSTOMER -> { + customersHierarchy.remove(entityId); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void addOrUpdateAttribute(AttributeKv attributeKv) { + var entityData = getOrCreate(attributeKv.getEntityId()); + if (entityData != null) { + KvEntry value = attributeKv.getValue(); + Integer keyId = KeyDictionary.get(attributeKv.getKey()); + boolean added = entityData.putAttr(keyId, attributeKv.getScope(), toDataPoint(attributeKv.getLastUpdateTs(), value)); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeAttribute(AttributeKv attributeKv) { + var entityData = get(attributeKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeAttr(KeyDictionary.get(attributeKv.getKey()), attributeKv.getScope()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); + } + } + } + + public void addOrUpdateLatestKv(LatestTsKv latestTsKv) { + var entityData = getOrCreate(latestTsKv.getEntityId()); + if (entityData != null) { + KvEntry value = latestTsKv.getValue(); + Integer keyId = KeyDictionary.get(latestTsKv.getKey()); + boolean added = entityData.putTs(keyId, toDataPoint(latestTsKv.getTs(), value)); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeLatestKv(LatestTsKv latestTsKv) { + var entityData = get(latestTsKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeTs(KeyDictionary.get(latestTsKv.getKey())); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); + } + } + } + + private DataPoint toDataPoint(long ts, KvEntry kvEntry) { + return switch (kvEntry.getDataType()) { + case BOOLEAN -> new BoolDataPoint(ts, kvEntry.getBooleanValue().get()); + case STRING -> getStrDataPoint(ts, kvEntry.getStrValue().get()); + case LONG -> new LongDataPoint(ts, kvEntry.getLongValue().get()); + case DOUBLE -> new DoubleDataPoint(ts, kvEntry.getDoubleValue().get()); + case JSON -> getJsonDataPoint(ts, kvEntry.getJsonValue().get()); + }; + } + + public void processFields(EntityFields fields) { + if (fields instanceof AssetFields assetFields) { + assetFields.setType(TbStringPool.intern(assetFields.getType())); + } + } + + private static DataPoint getStrDataPoint(long ts, String strV) { + DataPoint dp; + if (strV.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + dp = new StringDataPoint(ts, strV); + } else { + dp = new CompressedStringDataPoint(ts, strV); + } + return dp; + } + + private static DataPoint getJsonDataPoint(long ts, String strV) { + DataPoint dp; + if (strV.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + dp = new JsonDataPoint(ts, strV); + } else { + dp = new CompressedJsonDataPoint(ts, strV); + } + return dp; + } + + public ConcurrentMap> getEntityMap(EntityType entityType) { + return entityMapByType.computeIfAbsent(entityType, et -> new ConcurrentHashMap<>()); + } + + //TODO: automatically remove entities that has nothing except the ID. + private EntityData getOrCreate(EntityId entityId) { + return getOrCreate(entityId.getEntityType(), entityId.getId()); + } + + private EntityData getOrCreate(EntityType entityType, UUID entityId) { + return getEntityMap(entityType).computeIfAbsent(entityId, id -> { + log.debug("[{}] Adding {} {}", tenantId, entityType, id); + EntityData entityData = constructEntityData(entityType, entityId); + getEntitySet(entityType).add(entityData); + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); + return entityData; + }); + } + + private EntityData get(EntityId entityId) { + return getEntityMap(entityId.getEntityType()).get(entityId.getId()); + } + + private EntityData constructEntityData(EntityType entityType, UUID id) { + EntityData entityData = switch (entityType) { + case DEVICE -> new DeviceData(id); + case ASSET -> new AssetData(id); + case DEVICE_PROFILE, ASSET_PROFILE -> new EntityProfileData(id, entityType); + case CUSTOMER -> new CustomerData(id); + case TENANT -> new TenantData(id); + case ENTITY_GROUP -> new EntityGroupData(id); + case API_USAGE_STATE -> new ApiUsageStateData(id); + default -> new GenericData(entityType, id); + }; + entityData.setRepo(this); + return entityData; + } + + private static boolean entityIdMismatch(UUID oldOrNull, UUID newOrNull) { + if (oldOrNull == null) { + return newOrNull != null; + } else { + return !oldOrNull.equals(newOrNull); + } + } + + public Set> getEntitySet(EntityType entityType) { + return entitySetByType.computeIfAbsent(entityType, et -> new ConcurrentSkipListSet<>(CREATED_TIME_AND_ID_DESC_COMPARATOR)); + } + + public PageData findEntityDataByQuery(CustomerId customerId, MergedUserPermissions userPermissions, + EntityDataQuery oldQuery, boolean ignorePermissionCheck) { + EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, userPermissions, query.getEntityFilter(), ignorePermissionCheck); + if (ctx == null) { + return PageData.emptyPageData(); + } + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return sortAndConvert(query, queryProcessor.processQuery(), ctx); + } + + public long countEntitiesByQuery(CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery oldQuery, boolean ignorePermissionCheck) { + EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, userPermissions, query.getEntityFilter(), ignorePermissionCheck); + if (ctx == null) { + return 0; + } + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return queryProcessor.count(); + } + + private PageData sortAndConvert(EdqsDataQuery query, List data, QueryContext ctx) { + int totalSize = data.size(); + int totalPages = (int) Math.ceil((float) totalSize / query.getPageSize()); + int offset = query.getPage() * query.getPageSize(); + if (offset > totalSize) { + return new PageData<>(Collections.emptyList(), totalPages, totalSize, false); + } else { + Comparator comparator = EntityDataSortOrder.Direction.ASC.equals(query.getSortDirection()) ? SORT_ASC : SORT_DESC; + long startTs = System.nanoTime(); +// IMPLEMENTATION THAT IS BASED ON PRIORITY_QUEUE +// var requiredSize = Math.min(offset + query.getPageSize(), totalSize); +// PriorityQueue topN = new PriorityQueue<>(requiredSize, comparator.reversed()); +// for (SortableEntityData item : data) { +// topN.add(item); +// if (topN.size() > requiredSize) { +// topN.poll(); +// } +// } +// List result = new ArrayList<>(topN); +// Collections.reverse(result); +// result = result.subList(offset, requiredSize); +// IMPLEMENTATION THAT IS BASED ON TREE SET (For offset + query.getPageSize() << totalSize) + var requiredSize = Math.min(offset + query.getPageSize(), totalSize); + TreeSet topNSet = new TreeSet<>(comparator); + for (SortableEntityData sp : data) { + topNSet.add(sp); + if (topNSet.size() > requiredSize) { + topNSet.pollLast(); + } + } + var result = topNSet.stream().skip(offset).limit(query.getPageSize()).collect(Collectors.toList()); +// IMPLEMENTATION THAT IS BASED ON TIM SORT (For offset + query.getPageSize() > totalSize / 2) +// data.sort(comparator); +// var result = data.subList(offset, endIndex); + log.debug("EDQ Sorted in {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTs)); + return new PageData<>(toQueryResult(result, query, ctx), totalPages, totalSize, totalSize > requiredSize); + } + } + + private List toQueryResult(List data, EdqsDataQuery query, QueryContext ctx) { + long ts = System.currentTimeMillis(); + List results = new ArrayList<>(data.size()); + for (SortableEntityData entityData : data) { + Map> latest = new HashMap<>(); + for (var key : query.getEntityFields()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(EntityKeyType.ENTITY_FIELD, t -> new HashMap<>()).put(key.key(), v); + } + for (var key : query.getLatestValues()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(key.type(), t -> new HashMap<>()).put(KeyDictionary.get(key.keyId()), v); + } + + results.add(new QueryResult(entityData.getEntityId(), entityData.isReadAttrs(), entityData.isReadTs(), latest)); + } + return results; + } + + private QueryContext buildContext(CustomerId customerId, MergedUserPermissions userPermissions, EntityFilter filter, boolean ignorePermissionCheck) { + QueryContext queryContext; + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), SYS_ADMIN_PERMISSIONS, filter, ignorePermissionCheck); + } else { + switch (filter.getType()) { + case STATE_ENTITY_OWNER: + var singleEntity = ((StateEntityOwnerFilter) filter).getSingleEntity(); + EntityData ed = getEntityMap(singleEntity.getEntityType()).get(singleEntity.getId()); + if (ed != null) { + EntityId owner = ed.getCustomerId() != null ? new CustomerId(ed.getCustomerId()) : tenantId; + queryContext = new QueryContext(tenantId, customerId, owner.getEntityType(), userPermissions, filter, owner, ignorePermissionCheck); + } else { + return null; + } + break; + case SINGLE_ENTITY: + SingleEntityFilter seFilter = (SingleEntityFilter) filter; + EntityId entityId = seFilter.getSingleEntity(); + if (entityId != null && entityId.getEntityType().equals(EntityType.ENTITY_GROUP)) { + EntityGroupData entityGroupData = entityGroups.get(entityId.getId()); + if (entityGroupData != null) { + queryContext = new QueryContext(tenantId, customerId, EntityType.ENTITY_GROUP, userPermissions, filter, entityGroupData.getEntityType(), ignorePermissionCheck); + } else { + return null; + } + } else { + queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), userPermissions, filter, ignorePermissionCheck); + } + break; + default: + queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), userPermissions, filter, ignorePermissionCheck); + } + } + return queryContext; + } + + public TenantId getTenantId() { + return tenantId; + } + + public Set getAllCustomers(UUID customerId) { + Set result = new HashSet<>(); + Queue queue = new LinkedList<>(); + + if (customerId != null) { + queue.add(customerId); + } + + while (!queue.isEmpty()) { + UUID current = queue.poll(); + if (!result.contains(current)) { + result.add(current); + Set children = customersHierarchy.get(current); + if (children != null) { + queue.addAll(children); + } + } + } + + return result; + } + + public boolean contains(UUID entityGroupID, UUID entityId) { + var groupData = entityGroups.get(entityGroupID); + return groupData != null && groupData.getEntity(entityId) != null; + } + + public EntityGroupData getEntityGroup(UUID groupId) { + return entityGroups.get(groupId); + } + + public RelationsRepo getRelations(RelationTypeGroup relationTypeGroup) { + return relations.get(relationTypeGroup); + } + + public String getOwnerName(EntityId ownerId) { + if (ownerId == null || (EntityType.CUSTOMER.equals(ownerId.getEntityType()) && CustomerId.NULL_UUID.equals(ownerId.getId()))) { + ownerId = tenantId; + } + return getEntityName(ownerId); + } + + private String getEntityName(EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + return switch (entityType) { + case CUSTOMER, TENANT -> getEntityMap(entityType).get(entityId.getId()).getFields().getName(); + default -> throw new RuntimeException("Unsupported entity type: " + entityType); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java new file mode 100644 index 0000000000..e36e5ae5ab --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; + +import java.util.Set; + +public interface EdqsStateService { + + void restore(Set partitions); + + void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java new file mode 100644 index 0000000000..e8e8ee610e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@RequiredArgsConstructor +@KafkaEdqsComponent +@Slf4j +public class KafkaEdqsStateService implements EdqsStateService { + + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + private final EdqsQueueFactory queueFactory; + private final EdqsProcessor edqsProcessor; + + private MainQueueConsumerManager, QueueConfig> stateConsumer; + private QueueConsumerManager> eventsConsumer; + private EdqsProducer stateProducer; + + private ExecutorService consumersExecutor; + private ExecutorService mgmtExecutor; + private ScheduledExecutorService scheduler; + + private final VersionsStore versionsStore = new VersionsStore(); + private final AtomicInteger restoredCount = new AtomicInteger(); + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-backup-consumer")); + mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); + + stateConsumer = MainQueueConsumerManager., QueueConfig>builder() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) + .config(QueueConfig.of(true, config.getPollInterval())) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + edqsProcessor.process(msg, EdqsQueue.STATE); + if (restoredCount.incrementAndGet() % 1000 == 0) { + log.info("Processed {} msgs", restoredCount.get()); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.STATE)) + .consumerExecutor(consumersExecutor) + .taskExecutor(mgmtExecutor) + .scheduler(scheduler) + .build(); + + eventsConsumer = QueueConsumerManager.>builder() + .name("edqs-events-to-backup-consumer") + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + + if (msg.hasEventMsg()) { + EdqsEventMsg eventMsg = msg.getEventMsg(); + String key = eventMsg.getKey(); + if (eventMsg.hasVersion()) { + if (!versionsStore.isNew(key, eventMsg.getVersion())) { + return; + } + } + + TenantId tenantId = getTenantId(msg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + stateProducer.send(tenantId, objectType, key, msg); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group + .consumerExecutor(consumersExecutor) + .threadPrefix("edqs-events-to-backup") + .build(); + + stateProducer = EdqsProducer.builder() + .queue(EdqsQueue.STATE) + .partitionService(partitionService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.STATE)) + .build(); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + eventsConsumer.subscribe(); + eventsConsumer.launch(); + } + + @Override + public void restore(Set partitions) { + restoredCount.set(0); + long startTs = System.currentTimeMillis(); + log.info("Restore started for partitions {}", partitions.stream().map(tpi -> tpi.getPartition().orElse(-1)).sorted().toList()); + + stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update + stateConsumer.awaitStop(0); // consumers should stop on their own because EdqsQueue.STATE.stopWhenRead is true, we just need to wait + + log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), restoredCount.get()); + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + // do nothing here, backup is done by events consumer + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + @PreDestroy + private void preDestroy() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + eventsConsumer.stop(); + stateProducer.stop(); + + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + scheduler.shutdownNow(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java new file mode 100644 index 0000000000..8786f1866c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.util.EdqsRocksDb; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.util.Set; + +@Service +@RequiredArgsConstructor +@InMemoryEdqsComponent +@Slf4j +public class LocalEdqsStateService implements EdqsStateService { + + @Autowired @Lazy + private EdqsProcessor processor; + @Autowired + private EdqsRocksDb db; + + private Set partitions; + + @Override + public void restore(Set partitions) { + if (this.partitions == null) { + this.partitions = partitions; + } else { + return; + } + + db.forEach((key, value) -> { + try { + ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); + log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg); + processor.process(edqsMsg, EdqsQueue.STATE); + } catch (Exception e) { + log.error("[{}] Failed to restore value", key, e); + } + }); + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + log.trace("Save to RocksDB: {} {} {} {}", tenantId, type, key, msg); + try { + if (eventType == EdqsEventType.DELETED) { + db.delete(key); + } else { + db.put(key, msg.toByteArray()); + } + } catch (Exception e) { + log.error("[{}] Failed to save event {}", key, msg, e); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java new file mode 100644 index 0000000000..cfca43b08c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.stats; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@EdqsComponent +@Service +@Slf4j +@RequiredArgsConstructor +@ConditionalOnProperty(name = "queue.edqs.stats.enabled", havingValue = "true", matchIfMissing = true) +public class EdqsStatsService { + + private final ConcurrentHashMap statsMap = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + @Scheduled(initialDelayString = "${queue.edqs.stats.print-interval-ms:60000}", + fixedDelayString = "${queue.edqs.stats.print-interval-ms:60000}") + private void reportStats() { + String values = statsMap.entrySet().stream() + .map(kv -> "TenantId [" + kv.getKey() + "] stats [" + kv.getValue() + "]") + .collect(Collectors.joining(System.lineSeparator())); + log.info("EDQS Stats: {}", values); + } + + public void reportTenantEdqsObject(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { + statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) + .reportEdqsObject(objectType, eventType); + } + + @Getter + @AllArgsConstructor + static class EdqsStats { + + private final TenantId tenantId; + private final ConcurrentHashMap entityCounters = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + private AtomicInteger getOrCreateObjectCounter(ObjectType objectType) { + return entityCounters.computeIfAbsent(objectType, + type -> statsFactory.createGauge(StatsType.EDQS.getName() + "_object_count", new AtomicInteger(), + "tenantId", tenantId.toString(), "objectType", type.name())); + } + + @Override + public String toString() { + return entityCounters.entrySet().stream() + .map(counters -> counters.getKey().name()+ " total = [" + counters.getValue() + "]") + .collect(Collectors.joining(", ")); + } + + public void reportEdqsObject(ObjectType objectType, EdqsEventType eventType) { + AtomicInteger objectCounter = getOrCreateObjectCounter(objectType); + if (eventType == EdqsEventType.UPDATED){ + objectCounter.incrementAndGet(); + } else if (eventType == EdqsEventType.DELETED) { + objectCounter.decrementAndGet(); + } + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java new file mode 100644 index 0000000000..e8c260a996 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; + +@Service +@RequiredArgsConstructor +public class EdqsPartitionService { + + private final HashPartitionService hashPartitionService; + private final EdqsConfig edqsConfig; + + public Integer resolvePartition(TenantId tenantId) { + if (edqsConfig.getPartitioningStrategy() == EdqsPartitioningStrategy.TENANT) { + return hashPartitionService.resolvePartitionIndex(tenantId.getId(), edqsConfig.getPartitions()); + } else { + return null; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java new file mode 100644 index 0000000000..933d293c79 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import org.rocksdb.Options; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.nio.file.Files; +import java.nio.file.Path; + +@Component +@InMemoryEdqsComponent +public class EdqsRocksDb extends TbRocksDb { + + @Getter + private boolean isNew; + + public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${java.io.tmpdir}/edqs-backup}") String path) { + super(path, new Options().setCreateIfMissing(true)); + } + + @PostConstruct + public void init() { + isNew = !Files.exists(Path.of(path)); + super.init(); + } + + @PreDestroy + public void close() { + super.close(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java new file mode 100644 index 0000000000..1ad2634c1b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -0,0 +1,424 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.permission.MergedGroupTypePermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.EntitiesByGroupNameFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityGroupFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsCountQuery; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.KeyDictionary; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; +import static org.thingsboard.server.common.data.StringUtils.equalsAny; +import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithoutQuotes; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.AND; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.OR; + +@Slf4j +public class RepositoryUtils { + + public static final Comparator SORT_ASC = Comparator.comparing(SortableEntityData::getSortValue) + .thenComparing(sp -> sp.getId().toString()); + + public static final Comparator SORT_DESC = Comparator.comparing(SortableEntityData::getSortValue) + .thenComparing(sp -> sp.getId().toString()).reversed(); + + public static final MergedUserPermissions SYS_ADMIN_PERMISSIONS = new MergedUserPermissions(Collections.singletonMap(Resource.ALL, Set.of(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY)), Collections.emptyMap()); + public static final MergedUserPermissions ALL_READ_PERMISSIONS = new MergedUserPermissions( + Collections.singletonMap(Resource.ALL, Set.of(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY)), Collections.emptyMap()); + + public static EntityType resolveEntityType(EntityFilter entityFilter) { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> ((SingleEntityFilter) entityFilter).getSingleEntity().getEntityType(); + case ENTITY_GROUP -> ((EntityGroupFilter) entityFilter).getGroupType(); + case ENTITY_LIST -> ((EntityListFilter) entityFilter).getEntityType(); + case ENTITY_NAME -> ((EntityNameFilter) entityFilter).getEntityType(); + case ENTITY_TYPE -> ((EntityTypeFilter) entityFilter).getEntityType(); + case ENTITY_GROUP_LIST, ENTITY_GROUP_NAME -> EntityType.ENTITY_GROUP; + case ENTITIES_BY_GROUP_NAME -> ((EntitiesByGroupNameFilter) entityFilter).getGroupType(); + case STATE_ENTITY_OWNER -> throw new RuntimeException("Not implemented!"); // TODO: implement + case ASSET_TYPE, ASSET_SEARCH_QUERY -> EntityType.ASSET; + case DEVICE_TYPE, DEVICE_SEARCH_QUERY -> EntityType.DEVICE; + case ENTITY_VIEW_TYPE, ENTITY_VIEW_SEARCH_QUERY -> EntityType.ENTITY_VIEW; + case EDGE_TYPE, EDGE_SEARCH_QUERY -> EntityType.EDGE; + case RELATIONS_QUERY -> { + RelationsQueryFilter rgf = (RelationsQueryFilter) entityFilter; + yield rgf.isMultiRoot() ? rgf.getMultiRootEntitiesType() : rgf.getRootEntity().getEntityType(); + } + case API_USAGE_STATE -> EntityType.API_USAGE_STATE; + case SCHEDULER_EVENT -> EntityType.SCHEDULER_EVENT; + }; + } + + public static boolean hasNoPermissionsForAllRelationQueryResources(Map permissionsMap) { + if (permissionsMap.get(Resource.resourceFromEntityType(EntityType.TENANT)).isHasGenericRead()) { + return false; + } + for (EntityType entityType : EntityGroup.groupTypes) { + if (permissionsMap.get(Resource.resourceFromEntityType(entityType)).isHasGenericRead() || + !permissionsMap.get(Resource.resourceFromEntityType(entityType)).getEntityGroupIds().isEmpty()) { + return false; + } + if (permissionsMap.get(Resource.groupResourceFromGroupType(entityType)).isHasGenericRead() || + !permissionsMap.get(Resource.groupResourceFromGroupType(entityType)).getEntityGroupIds().isEmpty()) { + return false; + } + } + return true; + } + + public static boolean customerUserIsTryingToAccessTenantEntity(QueryContext ctx, EntityFilter entityFilter) { + if (ctx.isTenantUser()) { + return false; + } else { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> { + SingleEntityFilter seFilter = (SingleEntityFilter) entityFilter; + yield isSystemOrTenantEntity(seFilter.getSingleEntity().getEntityType()); + } + case ENTITY_LIST -> { + EntityListFilter elFilter = (EntityListFilter) entityFilter; + yield isSystemOrTenantEntity(elFilter.getEntityType()); + } + case ENTITY_NAME -> { + EntityNameFilter enFilter = (EntityNameFilter) entityFilter; + yield isSystemOrTenantEntity(enFilter.getEntityType()); + } + case ENTITY_TYPE -> { + EntityTypeFilter etFilter = (EntityTypeFilter) entityFilter; + yield isSystemOrTenantEntity(etFilter.getEntityType()); + } + default -> false; + }; + } + } + + private static boolean isSystemOrTenantEntity(EntityType entityType) { + return switch (entityType) { + case INTEGRATION, CONVERTER, DEVICE_PROFILE, ASSET_PROFILE, RULE_CHAIN, SCHEDULER_EVENT, TENANT, + TENANT_PROFILE, WIDGET_TYPE, WIDGETS_BUNDLE -> true; + default -> false; + }; + } + + public static EdqsDataQuery toNewQuery(EntityDataQuery oldQuery) { + var query = EdqsDataQuery.builder(); + query.page(oldQuery.getPageLink().getPage()); + query.pageSize(oldQuery.getPageLink().getPageSize()); + query.textSearch(oldQuery.getPageLink().getTextSearch()); + var sortOrder = oldQuery.getPageLink().getSortOrder(); + if (sortOrder != null && toNewKey(sortOrder.getKey()) != null) { + query.sortKey(toNewKey(sortOrder.getKey())); + query.sortDirection(sortOrder.getDirection()); + } else { + query.sortKey(new DataKey(EntityKeyType.ENTITY_FIELD, "createdTime", null)); + query.sortDirection(EntityDataSortOrder.Direction.DESC); + } + query.entityFilter(oldQuery.getEntityFilter()); + query.keyFilters(toKeyFilters(oldQuery.getKeyFilters())); + query.entityFields(toNewKeys(oldQuery.getEntityFields())); + query.latestValues(toNewKeys(oldQuery.getLatestValues())); + return query.build(); + } + + public static EdqsCountQuery toNewQuery(EntityCountQuery oldQuery) { + return EdqsCountQuery.builder() + .entityFilter(oldQuery.getEntityFilter()) + .hasKeyFilters(CollectionsUtil.isNotEmpty(oldQuery.getKeyFilters())) + .keyFilters(toKeyFilters(oldQuery.getKeyFilters())) + .build(); + } + + private static List toKeyFilters(List keyFilters) { + if (keyFilters == null || keyFilters.isEmpty()) { + return Collections.emptyList(); + } else { + List result = new ArrayList<>(); + for (KeyFilter entityFilter : keyFilters) { + var newKey = toNewKey(entityFilter.getKey()); + if (newKey != null) { + result.add(new EdqsFilter(newKey, entityFilter.getValueType(), entityFilter.getPredicate())); + } + } + return result; + } + } + + private static DataKey toNewKey(EntityKey entityKey) { + if (EntityKeyType.ENTITY_FIELD.equals(entityKey.getType())) { + return new DataKey(entityKey.getType(), entityKey.getKey(), null); + } + Integer keyId = KeyDictionary.get(entityKey.getKey()); + if (keyId != null) { + return new DataKey(entityKey.getType(), entityKey.getKey(), keyId); + } else { + log.warn("Missing dictionary key for {}", entityKey.getKey()); + return null; + } + } + + private static List toNewKeys(List entityKeys) { + if (entityKeys == null || entityKeys.isEmpty()) { + return Collections.emptyList(); + } else { + var result = new ArrayList(entityKeys.size()); + for (EntityKey entityKey : entityKeys) { + var newKey = toNewKey(entityKey); + if (newKey != null) { + result.add(newKey); + } + } + return result; + } + } + + public static boolean checkKeyFilters(EntityData entity, List keyFilters) { + for (EdqsFilter keyFilter : keyFilters) { + EntityKeyValueType valueType = keyFilter.valueType(); + if (valueType == null) { + valueType = switch (keyFilter.predicate().getType()) { + case STRING -> EntityKeyValueType.STRING; + case NUMERIC -> EntityKeyValueType.NUMERIC; + case BOOLEAN -> EntityKeyValueType.BOOLEAN; + default -> throw new IllegalStateException(); + }; + } + DataPoint dp = entity.getDataPoint(keyFilter.key(), null); + boolean checkResult = switch (valueType) { + case STRING -> { + String str = dp != null ? dp.valueToString() : null; + yield StringUtils.isEmpty(str) || checkKeyFilter(str, keyFilter.predicate()); + } + case BOOLEAN -> { + Boolean booleanValue = dp != null ? dp.getBool() : null; + yield booleanValue != null && checkKeyFilter(booleanValue, keyFilter.predicate()); + } + case DATE_TIME, NUMERIC -> { + Double doubleValue = dp != null ? dp.getDouble() : null; + yield doubleValue != null && checkKeyFilter(doubleValue, keyFilter.predicate()); + } + }; + if (!checkResult) { + return false; + } + } + return true; + } + + public static boolean checkKeyFilter(String value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.STRING) { + throw new IllegalStateException("Not implemented"); + } + StringFilterPredicate predicate = (StringFilterPredicate) keyFilterPredicate; + String predicateValue = predicate.getValue().getValue(); + if (StringUtils.isEmpty(predicateValue)) { + return true; + } + if (predicate.isIgnoreCase()) { + predicateValue = predicateValue.toLowerCase(); + value = value.toLowerCase(); + } + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case STARTS_WITH -> value.startsWith(predicateValue); + case ENDS_WITH -> value.endsWith(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case CONTAINS -> value.contains(predicateValue); + case NOT_CONTAINS -> !value.contains(predicateValue); + case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + }; + } + + public static boolean checkKeyFilter(Double value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.NUMERIC) { + throw new IllegalStateException("Not implemented"); + } + NumericFilterPredicate predicate = (NumericFilterPredicate) keyFilterPredicate; + Double predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case GREATER -> value.compareTo(predicateValue) > 0; + case LESS -> value.compareTo(predicateValue) < 0; + case GREATER_OR_EQUAL -> value.compareTo(predicateValue) >= 0; + case LESS_OR_EQUAL -> value.compareTo(predicateValue) <= 0; + }; + } + + public static boolean checkKeyFilter(Boolean value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.BOOLEAN) { + throw new IllegalStateException("Not implemented"); + } + BooleanFilterPredicate predicate = (BooleanFilterPredicate) keyFilterPredicate; + Boolean predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + }; + } + + public static boolean checkComplexKeyFilter(T value, ComplexFilterPredicate filterPredicates, + SimpleKeyFilter simpleKeyFilter) { + if (filterPredicates.getOperation() == AND) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (!simpleKeyFilter.check(value, filterPredicate)) { + return false; + } + } + return true; + } else if (filterPredicates.getOperation() == OR) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (simpleKeyFilter.check(value, filterPredicate)) { + return true; + } + } + return false; + } else { + return false; + } + } + + public static Pattern toSqlLikePattern(String nameFilter) { + if (StringUtils.isNotBlank(nameFilter)) { + boolean percentSymbolOnStart = nameFilter.startsWith("%"); + boolean percentSymbolOnEnd = nameFilter.endsWith("%"); + if (percentSymbolOnStart) { + nameFilter = nameFilter.substring(1); + } + if (percentSymbolOnEnd) { + nameFilter = nameFilter.substring(0, nameFilter.length() - 1); + } + if (percentSymbolOnStart || percentSymbolOnEnd) { + return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE); + } + } + return null; + } + + @FunctionalInterface + public interface SimpleKeyFilter { + + boolean check(T value, KeyFilterPredicate predicate); + + } + + public static TsValue toTsValue(long ts, DataPoint dp) { + if (dp != null) { + return new TsValue(dp.getTs() > 0 ? dp.getTs() : ts, dp.valueToString()); + } else { + return new TsValue(ts, ""); + } + } + + public static String getSortValue(EntityData entity, DataKey sortKey) { + if (sortKey == null) { + return null; + } + switch (sortKey.type()) { + case ENTITY_FIELD -> { + return entity.getField(sortKey.key()); + } + case ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE, SERVER_ATTRIBUTE -> { + var dp = entity.getAttr(sortKey.keyId(), sortKey.type()); + return dp != null ? dp.valueToString() : ""; + } + case TIME_SERIES -> { + var dp = entity.getTs(sortKey.keyId()); + return dp != null ? dp.valueToString() : ""; + } + default -> throw new IllegalStateException("toSortKey is not implemented for type: " + sortKey.type()); + } + } + + public static boolean checkFilters(EdqsQuery query, EntityData entity) { + if (entity == null || entity.getFields() == null) { + return false; // Entity was already removed or not arrived yet; + } + if (query.isHasKeyFilters() && !checkKeyFilters(entity, query.getKeyFilters())) { + return false; + } + if (query instanceof EdqsDataQuery dataQuery) { + return !dataQuery.isHasTextSearch() || checkTextSearch(entity, dataQuery); + } + return true; + } + + private static boolean checkTextSearch(EntityData entityData, EdqsDataQuery query) { + return Stream.concat(query.getEntityFields().stream(), query.getLatestValues().stream()) + .anyMatch(key -> { + DataPoint value = entityData.getDataPoint(key, null); + return value != null && containsIgnoreCase(value.valueToString(), query.getTextSearch()); + }); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java new file mode 100644 index 0000000000..9b06a79a1f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import java.nio.charset.StandardCharsets; +import java.util.function.BiConsumer; + +@RequiredArgsConstructor +public class TbRocksDb { + + protected final String path; + private final Options options; + + private RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + @SneakyThrows + public void init() { + db = RocksDB.open(options, path); + } + + public void put(String key, byte[] value) throws RocksDBException { + db.put(key.getBytes(StandardCharsets.UTF_8), value); + } + + public void forEach(BiConsumer processor) { + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + processor.accept(key, iterator.value()); + } + } + } + + public void delete(String key) throws RocksDBException { + db.delete(key.getBytes(StandardCharsets.UTF_8)); + } + + public void close() { + if (db != null) { + db.close(); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java new file mode 100644 index 0000000000..de2c5a1955 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class VersionsStore { + + private final Cache versions = Caffeine.newBuilder() + .expireAfterWrite(1, TimeUnit.HOURS) + .build(); + + public boolean isNew(String key, Long version) { + AtomicBoolean isNew = new AtomicBoolean(false); + versions.asMap().compute(key, (k, prevVersion) -> { + if (prevVersion == null || prevVersion < version) { + isNew.set(true); + return version; + } else { + if (version < prevVersion) { + log.info("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion); + } + return prevVersion; + } + }); + return isNew.get(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java new file mode 100644 index 0000000000..9ca7d86e0a --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.edqs; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EdqsService { + + ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request); + + boolean isApiEnabled(); + + void onUpdate(TenantId tenantId, EntityId entityId, Object entity); + + void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void onDelete(TenantId tenantId, EntityId entityId); + + void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void processSystemRequest(ToCoreEdqsRequest request); + + void processSystemMsg(ToCoreEdqsMsg request); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index f3a0e47d09..ce2d31a745 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -26,7 +26,8 @@ public enum ServiceType { TB_RULE_ENGINE("TB Rule Engine"), TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), - TB_VC_EXECUTOR("TB VC Executor"); + TB_VC_EXECUTOR("TB VC Executor"), + EDQS("TB Entity Data Query Service"); private final String label; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index cbddc83b2b..552bdf50d6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -30,26 +30,33 @@ public class TopicPartitionInfo { private final TenantId tenantId; private final Integer partition; @Getter + private final boolean useInternalPartition; + @Getter private final String fullTopicName; @Getter private final boolean myPartition; @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean useInternalPartition, boolean myPartition) { this.topic = topic; this.tenantId = tenantId; this.partition = partition; + this.useInternalPartition = useInternalPartition; this.myPartition = myPartition; String tmp = topic; if (tenantId != null && !tenantId.isNullUid()) { tmp += ".isolated." + tenantId.getId().toString(); } - if (partition != null) { + if (partition != null && !useInternalPartition) { tmp += "." + partition; } this.fullTopicName = tmp; } + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + this(topic, tenantId, partition, false, myPartition); + } + public TopicPartitionInfo newByTopic(String topic) { return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); } @@ -66,6 +73,14 @@ public class TopicPartitionInfo { return Optional.ofNullable(partition); } + public TopicPartitionInfo withTopic(String topic) { + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); + } + + public TopicPartitionInfo withUseInternalPartition(boolean useInternalPartition) { + return new TopicPartitionInfo(this.topic, this.tenantId, this.partition, useInternalPartition, this.myPartition); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -79,6 +94,7 @@ public class TopicPartitionInfo { @Override public int hashCode() { - return Objects.hash(fullTopicName); + return Objects.hash(fullTopicName, partition); } + } diff --git a/common/pom.xml b/common/pom.xml index 4c8f56cbce..9cb67ec989 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -49,6 +49,7 @@ edge-api version-control script + edqs diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 228a4039d2..d21862d3e1 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -70,6 +70,7 @@ message ServiceInfo { repeated string transports = 6; SystemInfoProto systemInfo = 10; repeated string assignedTenantProfiles = 11; + string label = 12; } message SystemInfoProto { @@ -170,12 +171,33 @@ message AttributeValueProto { optional int64 version = 10; } +message AttributeKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + AttributeScopeProto scope = 4; + string key = 5; + int64 version = 6; + int64 lastUpdateTs = 7; + KeyValueProto value = 8; +} + message TsKvProto { int64 ts = 1; KeyValueProto kv = 2; optional int64 version = 3; } +message LatestTsKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + string key = 4; + int64 ts = 5; + int64 version = 6; + KeyValueProto value = 7; +} + message TsKvListProto { int64 ts = 1; repeated KeyValueProto kv = 2; @@ -482,6 +504,10 @@ message ImageCacheKeyProto { optional string publicResourceKey = 2; } +message ToEdqsCoreServiceMsg { + bytes value = 1; +} + message LwM2MRegistrationRequestMsg { string tenantId = 1; string endpoint = 2; @@ -1532,6 +1558,7 @@ message ToCoreNotificationMsg { ToEdgeSyncRequestMsgProto toEdgeSyncRequest = 11 [deprecated = true]; FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; + ToEdqsCoreServiceMsg toEdqsCoreServiceMsg = 17; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; } @@ -1660,3 +1687,33 @@ message HousekeeperTaskProto { int32 attempt = 50; repeated string errors = 51; } + +message ToEdqsMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 customerIdMSB = 3; + int64 customerIdLSB = 4; + int64 ts = 5; + EdqsEventMsg eventMsg = 6; + EdqsRequestMsg requestMsg = 7; +} + +message FromEdqsMsg { + EdqsResponseMsg responseMsg = 1; +} + +message EdqsEventMsg { + string key = 1; + string objectType = 2; + bytes data = 3; + string eventType = 4; + optional int64 version = 5; +} + +message EdqsRequestMsg { + string value = 1; +} + +message EdqsResponseMsg { + string value = 1; +} diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 44e2239e05..99ad24575e 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -116,6 +116,10 @@ org.apache.curator curator-recipes + + org.xerial.snappy + snappy-java + org.springframework.boot spring-boot-starter-test diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 9513565ca1..88aa2a233f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -95,9 +95,8 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - List topicNames = getFullTopicNames(); - log.info("Subscribing to topics {}", topicNames); - doSubscribe(topicNames); + log.info("Subscribing to topics {}", getFullTopicNames()); + doSubscribe(partitions); subscribed = true; } records = partitions.isEmpty() ? emptyList() : doPoll(durationInMillis); @@ -187,7 +186,7 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected T decode(R record) throws IOException; - abstract protected void doSubscribe(List topicNames); + abstract protected void doSubscribe(Set partitions); abstract protected void doCommit(); @@ -198,7 +197,10 @@ public abstract class AbstractTbQueueConsumerTemplate i if (partitions == null) { return Collections.emptyList(); } - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + return partitions.stream() + .map(tpi -> tpi.getFullTopicName() + (tpi.isUseInternalPartition() ? + "[" + tpi.getPartition().orElse(-1) + "]" : "")) + .collect(Collectors.toList()); } protected boolean isLongPollingSupported() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 950519b098..a2bdde6d66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -211,6 +211,15 @@ public class DefaultTbQueueRequestTemplate send(Request request, long requestTimeoutNs) { + return send(request, requestTimeoutNs, null); + } + + @Override + public ListenableFuture send(Request request, Integer partition) { + return send(request, this.maxRequestTimeoutNs, partition); + } + + private ListenableFuture send(Request request, long requestTimeoutNs, Integer partition) { if (pendingRequests.mappingCount() >= maxPendingRequests) { log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance. Request is {}", maxPendingRequests, request); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); @@ -227,7 +236,7 @@ public class DefaultTbQueueRequestTemplate future, ResponseMetaData responseMetaData) { + void sendToRequestTemplate(Request request, UUID requestId, Integer partition, SettableFuture future, ResponseMetaData responseMetaData) { log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } - requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(requestTemplate.getDefaultTopic()) + .partition(partition) + .useInternalPartition(partition != null) + .build(); + requestTemplate.send(tpi, request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { if (messagesStats != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index 1042eeb280..78f2085397 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -28,6 +28,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueResponseTemplate; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -77,9 +78,18 @@ public class DefaultTbQueueResponseTemplate handler) { - this.responseTemplate.init(); + public void subscribe() { requestTemplate.subscribe(); + } + + @Override + public void subscribe(Set partitions) { + requestTemplate.subscribe(partitions); + } + + @Override + public void launch(TbQueueHandler handler) { + this.responseTemplate.init(); loopExecutor.submit(() -> { while (!stopped) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 6eb5c94c9b..2f44a9c4ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.consumer; +package org.thingsboard.server.queue.common.consumer; import lombok.Builder; import lombok.Getter; @@ -24,9 +24,6 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.service.queue.ruleengine.QueueEvent; -import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerManagerTask; -import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerTask; import java.util.Collection; import java.util.Collections; @@ -47,6 +44,7 @@ import java.util.stream.Collectors; @Slf4j public class MainQueueConsumerManager { + @Getter protected final QueueKey queueKey; @Getter protected C config; @@ -182,7 +180,7 @@ public class MainQueueConsumerManager partitions) { + public void doUpdate(Set partitions) { this.partitions = partitions; consumerWrapper.updatePartitions(partitions); } @@ -236,13 +234,19 @@ public class MainQueueConsumerManager consumerTask.awaitCompletion(timeoutSec)); log.debug("[{}] Unsubscribed and stopped consumers", queueKey); } private static String partitionsToString(Collection partitions) { - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]")); + return partitions.stream().map(tpi -> tpi.getFullTopicName() + (tpi.isUseInternalPartition() ? + "[" + tpi.getPartition().orElse(-1) + "]" : "")) + .collect(Collectors.joining(", ", "[", "]")); } public interface MsgPackProcessor { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java index 9e5766b374..1a78cfc2ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.queue.common.consumer; import java.io.Serializable; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java index e5821df68d..67bf370db0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.queue.common.consumer; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java index 5e672eb5c6..708e24fdac 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.queue.common.consumer; import lombok.Getter; import lombok.Setter; @@ -70,10 +70,18 @@ public class TbQueueConsumerTask { } public void awaitCompletion() { + awaitCompletion(30); + } + + public void awaitCompletion(int timeoutSec) { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(30, TimeUnit.SECONDS); + if (timeoutSec > 0) { + task.get(timeoutSec, TimeUnit.SECONDS); + } else { + task.get(); + } log.trace("[{}] Awaited finish", key); } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 1ceb52b3d6..64677009ff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.util.AfterContextReady; import java.net.InetAddress; @@ -63,6 +64,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.rule_engine.assigned_tenant_profiles:}") private Set assignedTenantProfiles; + @Autowired + private EdqsConfig edqsConfig; + @Autowired private ApplicationContext applicationContext; @@ -87,6 +91,11 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (!serviceTypes.contains(ServiceType.TB_RULE_ENGINE) || assignedTenantProfiles == null) { assignedTenantProfiles = Collections.emptySet(); } + if (serviceTypes.contains(ServiceType.EDQS)) { + if (StringUtils.isBlank(edqsConfig.getLabel())) { + edqsConfig.setLabel(serviceId); + } + } generateNewServiceInfoWithCurrentSystemInfo(); } @@ -123,6 +132,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); } + builder.setLabel(edqsConfig.getLabel()); return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 37e519e3f2..165a54b300 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -20,6 +20,7 @@ import com.google.common.hash.Hashing; import jakarta.annotation.PostConstruct; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -70,6 +71,8 @@ public class HashPartitionService implements PartitionService { private String edgeTopic; @Value("${queue.edge.partitions:10}") private Integer edgePartitions; + @Value("${queue.edqs.partitions:12}") + private Integer edqsPartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @@ -123,6 +126,10 @@ public class HashPartitionService implements PartitionService { QueueKey edgeKey = coreKey.withQueueName(EDGE_QUEUE_NAME); partitionSizesMap.put(edgeKey, edgePartitions); partitionTopicsMap.put(edgeKey, edgeTopic); + + QueueKey edqsKey = new QueueKey(ServiceType.EDQS); + partitionSizesMap.put(edqsKey, edqsPartitions); + partitionTopicsMap.put(edqsKey, "edqs"); // placeholder, not used } @AfterStartUp(order = AfterStartUp.QUEUE_INFO_INITIALIZATION) @@ -211,7 +218,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); } } @@ -354,6 +361,11 @@ public class HashPartitionService implements PartitionService { } } + @Override + public boolean isSystemPartitionMine(ServiceType serviceType) { + return isMyPartition(serviceType, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); + } + @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { log.info("Recalculating partitions"); @@ -374,9 +386,9 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { try { - ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); - log.trace("Server responsible for {}[{}] - {}", queueKey, i, serviceInfo != null ? serviceInfo.getServiceId() : "none"); - if (currentService.equals(serviceInfo)) { + List services = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); + log.trace("Server responsible for {}[{}] - {}", queueKey, i, services); + if (services.contains(currentService)) { newPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); } } catch (Exception e) { @@ -390,6 +402,7 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); + Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -410,19 +423,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - changedPartitionsMap.put(queueKey, tpiList); + changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); + oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); } }); if (!changedPartitionsMap.isEmpty()) { - Map>> partitionsByServiceType = new HashMap<>(); - changedPartitionsMap.forEach((queueKey, partitions) -> { - partitionsByServiceType.computeIfAbsent(queueKey.getType(), serviceType -> new HashMap<>()) - .put(queueKey, partitions); - }); - partitionsByServiceType.forEach(this::publishPartitionChangeEvent); + changedPartitionsMap.entrySet().stream() + .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .forEach((serviceType, partitionsMap) -> { + publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); + }); } if (currentOtherServices == null) { @@ -454,13 +464,15 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { - log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { + log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions, oldPartitions); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -468,6 +480,15 @@ public class HashPartitionService implements PartitionService { } } + private Set toTpiList(QueueKey queueKey, List partitions) { + if (partitions == null) { + return null; + } + return partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + } + @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); @@ -598,6 +619,8 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(new QueueKey(serviceType).withQueueName(EDGE_QUEUE_NAME), key -> new ArrayList<>()).add(instance); } else if (ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); + } else if (ServiceType.EDQS.equals(serviceType)) { + queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } @@ -606,10 +629,11 @@ public class HashPartitionService implements PartitionService { } } - protected ServiceInfo resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, - Map> responsibleServices) { + @NotNull + protected List resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, + Map> responsibleServices) { if (servers == null || servers.isEmpty()) { - return null; + return Collections.emptyList(); } TenantId tenantId = queueKey.getTenantId(); @@ -637,15 +661,21 @@ public class HashPartitionService implements PartitionService { responsibleServices.put(profileId, responsible); } if (responsible.isEmpty()) { - return null; + return Collections.emptyList(); } servers = responsible; } int hash = hash(tenantId.getId()); - return servers.get(Math.abs((hash + partition) % servers.size())); + ServiceInfo server = servers.get(Math.abs((hash + partition) % servers.size())); + return server != null ? List.of(server) : Collections.emptyList(); + } else if (queueKey.getType() == ServiceType.EDQS) { + List> sets = servers.stream().collect(Collectors.groupingBy(ServiceInfo::getLabel)) + .entrySet().stream().sorted(Map.Entry.comparingByKey()).map(Map.Entry::getValue).toList(); + return sets.get(partition % sets.size()); } else { - return servers.get(partition % servers.size()); + ServiceInfo server = servers.get(partition % servers.size()); + return server != null ? List.of(server) : Collections.emptyList(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b5744981bd..a95456b5e5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -41,6 +41,8 @@ public interface PartitionService { boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); + boolean isSystemPartitionMine(ServiceType serviceType); + List getMyPartitions(QueueKey queueKey); /** @@ -61,8 +63,6 @@ public interface PartitionService { Set getOtherServices(ServiceType serviceType); - int resolvePartitionIndex(UUID entityId, int partitions); - void evictTenantInfo(TenantId tenantId); int countTransportsByType(String type); @@ -75,4 +75,6 @@ public interface PartitionService { boolean isManagedByCurrentService(TenantId tenantId); + int resolvePartitionIndex(UUID entityId, int partitions); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 3a8d365433..8107b7c3eb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -19,6 +19,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.ProtocolStringList; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; @@ -68,6 +69,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private Integer zkConnectionTimeout; @Value("${zk.session_timeout_ms}") private Integer zkSessionTimeout; + @Getter @Value("${zk.zk_dir}") private String zkDir; @Value("${zk.recalculate_delay:0}") @@ -80,6 +82,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private final PartitionService partitionService; private ScheduledExecutorService zkExecutorService; + @Getter private CuratorFramework client; private PathChildrenCache cache; private String nodePath; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 88ceb4aa08..e0e7db80a7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; +import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -36,12 +37,17 @@ public class PartitionChangeEvent extends TbApplicationEvent { @Getter private final ServiceType serviceType; @Getter - private final Map> partitionsMap; + private final Map> newPartitions; + @Getter + private final Map> oldPartitions; - public PartitionChangeEvent(Object source, ServiceType serviceType, Map> partitionsMap) { + public PartitionChangeEvent(Object source, ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { super(source); this.serviceType = serviceType; - this.partitionsMap = partitionsMap; + this.newPartitions = newPartitions; + this.oldPartitions = oldPartitions; } public Set getCorePartitions() { @@ -52,11 +58,16 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } + public Set getPartitions() { + return newPartitions.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); + } + private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { - return partitionsMap.entrySet() + return newPartitions.entrySet() .stream() .filter(entry -> serviceType.equals(entry.getKey().getType()) && queueName.equals(entry.getKey().getQueueName())) .flatMap(entry -> entry.getValue().stream()) .collect(Collectors.toSet()); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java new file mode 100644 index 0000000000..838f9b4aa2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +// TODO: tb-core ? +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local'))") +public @interface EdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java new file mode 100644 index 0000000000..f35827a8ce --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Data +public class EdqsConfig { + + @Value("${queue.edqs.partitions:12}") + private int partitions; + @Value("${service.edqs.label:}") + private String label; + @Value("#{'${queue.edqs.partitioning_strategy:tenant}'.toUpperCase()}") + private EdqsPartitioningStrategy partitioningStrategy; + + @Value("${queue.edqs.requests_topic:edqs.requests}") + private String requestsTopic; + @Value("${queue.edqs.responses_topic:edqs.responses}") + private String responsesTopic; + @Value("${queue.edqs.poll_interval:125}") + private long pollInterval; + @Value("${queue.edqs.max_pending_requests:10000}") + private int maxPendingRequests; + @Value("${queue.edqs.max_request_timeout:10000}") + private int maxRequestTimeout; + + public String getLabel() { + if (partitioningStrategy == EdqsPartitioningStrategy.NONE) { + label = "all"; // single set for all instances, so that each instance has all partitions + } + return label; + } + + public enum EdqsPartitioningStrategy { + TENANT, NONE + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java new file mode 100644 index 0000000000..c773ea4e93 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Getter; + +@Getter +public enum EdqsQueue { + + EVENTS("edqs.events", false, false), + STATE("edqs.state", true, true); + + private final String topic; + private final boolean readFromBeginning; // read from the beginning of the topic, instead of the latest committed offset + private final boolean stopWhenRead; // stop consuming when reached an empty msg pack + + EdqsQueue(String topic, boolean readFromBeginning, boolean stopWhenRead) { + this.topic = topic; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java new file mode 100644 index 0000000000..dc4a9d645c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface EdqsQueueFactory { + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue); + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group); + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java new file mode 100644 index 0000000000..5055787fde --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") +public @interface InMemoryEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java new file mode 100644 index 0000000000..20b4beadcf --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.DummyMessagesStats; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.memory.InMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; + +@Component +@InMemoryEdqsComponent +@RequiredArgsConstructor +public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { + + private final InMemoryStorage storage; + private final EdqsConfig edqsConfig; + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueConsumer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return createEdqsMsgConsumer(queue); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + TbQueueConsumer> requestConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueProducer> responseProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getResponsesTopic()); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer) + .responseTemplate(responseProducer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(new DummyMessagesStats()) // FIXME + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java new file mode 100644 index 0000000000..9f112e7f59 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='kafka'))") +public @interface KafkaEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java new file mode 100644 index 0000000000..4b599670a1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -0,0 +1,122 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.DummyMessagesStats; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; + +import java.util.concurrent.atomic.AtomicInteger; + +@Component +@KafkaEdqsComponent +public class KafkaEdqsQueueFactory implements EdqsQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbKafkaAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; + private final TbKafkaAdmin edqsStateAdmin; + private final EdqsConfig edqsConfig; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final AtomicInteger consumerCounter = new AtomicInteger(); + + public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs, + EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider, + TbKafkaConsumerStatsService consumerStatsService) { + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsRequestsConfigs()); + this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs()); + this.kafkaSettings = kafkaSettings; + this.edqsConfig = edqsConfig; + this.serviceInfoProvider = serviceInfoProvider; + this.consumerStatsService = consumerStatsService; + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + String consumerGroup = "edqs-" + queue.name().toLowerCase() + "-consumer-group-" + serviceInfoProvider.getServiceId(); + return createEdqsMsgConsumer(queue, consumerGroup); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(queue.getTopic()) + .readFromBeginning(queue.isReadFromBeginning()) + .stopWhenRead(queue.isStopWhenRead()) + .clientId("edqs-" + queue.name().toLowerCase() + "-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(group) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-" + queue.name().toLowerCase() + "-producer-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + String requestsConsumerGroup = "edqs-requests-consumer-group-" + edqsConfig.getLabel(); + var requestConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(edqsConfig.getRequestsTopic()) + .clientId("edqs-requests-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(requestsConsumerGroup) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + var responseProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(edqsConfig.getResponsesTopic()) + .admin(edqsRequestsAdmin); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer.build()) + .responseTemplate(responseProducer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(new DummyMessagesStats()) // FIXME + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java new file mode 100644 index 0000000000..b69b98dd4f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLock { + + void lock(); + + void unlock(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java new file mode 100644 index 0000000000..61a21cca7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLockService { + + DistributedLock getLock(String key); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java new file mode 100644 index 0000000000..03b6d0f2d9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.util.concurrent.locks.ReentrantLock; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +public class DummyDistributedLockService implements DistributedLockService { + + @Override + public DistributedLock getLock(String key) { + return new DummyDistributedLock<>(); + } + + @RequiredArgsConstructor + private static class DummyDistributedLock implements DistributedLock { + + private final ReentrantLock lock = new ReentrantLock(); + + @SneakyThrows + @Override + public void lock() { + lock.lock(); + } + + @SneakyThrows + @Override + public void unlock() { + lock.unlock(); + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java new file mode 100644 index 0000000000..07b8b503f9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.ZkDiscoveryService; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true") +@Slf4j +public class ZkDistributedLockService implements DistributedLockService { + + private final ZkDiscoveryService zkDiscoveryService; + + @Override + public DistributedLock getLock(String key) { + return new ZkDistributedLock<>(key); + } + + @RequiredArgsConstructor + private class ZkDistributedLock implements DistributedLock { + + private final InterProcessLock interProcessLock; + + public ZkDistributedLock(String key) { + this.interProcessLock = new InterProcessMutex(zkDiscoveryService.getClient(), zkDiscoveryService.getZkDir() + "/locks/" + key); + } + + @SneakyThrows + @Override + public void lock() { + interProcessLock.acquire(); + } + + @SneakyThrows + @Override + public void unlock() { + interProcessLock.release(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index a2edc35d94..2469e720bd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -28,7 +28,13 @@ public class KafkaTbQueueMsg implements TbQueueMsg { private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); + UUID key; + try { + key = UUID.fromString(record.key()); + } catch (IllegalArgumentException e) { + key = null; // FIXME + } + this.key = key; TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); record.headers().forEach(header -> { headers.put(header.key(), header.value()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 2ea11c7afa..461defa58f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -57,7 +57,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { String numPartitionsStr = topicConfigs.get(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); if (numPartitionsStr != null) { numPartitions = Integer.parseInt(numPartitionsStr); - topicConfigs.remove("partitions"); } else { numPartitions = 1; } @@ -71,7 +70,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { return; } try { - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(PropertyUtils.getProps(topicConfigs, properties)); + Map configs = PropertyUtils.getProps(topicConfigs, properties); + configs.remove(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(configs); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { @@ -188,6 +189,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { public boolean isTopicEmpty(String topic) { try { + if (!getTopics().contains(topic)) { + return true; + } TopicDescription topicDescription = settings.getAdminClient().describeTopics(Collections.singletonList(topic)).topicNameValues().get(topic).get(); List partitions = topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())).toList(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index ef79834735..167fbb1751 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -21,7 +21,9 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; import org.springframework.util.StopWatch; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; @@ -30,8 +32,12 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -46,10 +52,17 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final TbKafkaConsumerStatsService statsService; private final String groupId; + private final boolean readFromBeginning; // reset offset to beginning + private final boolean stopWhenRead; // stop consuming when reached an empty msg pack + private Map endOffsets; // needed if stopWhenRead is true + + private boolean partitionsAssigned = false; + @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, - TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) { + TbQueueAdmin admin, TbKafkaConsumerStatsService statsService, + boolean readFromBeginning, boolean stopWhenRead) { super(topic); Properties props = settings.toConsumerProps(topic); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); @@ -67,13 +80,45 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; } @Override - protected void doSubscribe(List topicNames) { - if (!topicNames.isEmpty()) { - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); + protected void doSubscribe(Set partitions) { + Map> topics; + if (partitions == null) { + topics = Collections.emptyMap(); + } else { + topics = new HashMap<>(); + partitions.forEach(tpi -> { + if (tpi.isUseInternalPartition()) { + topics.computeIfAbsent(tpi.getFullTopicName(), t -> new ArrayList<>()).add(tpi.getPartition().get()); + } else { + topics.put(tpi.getFullTopicName(), null); + } + }); + } + if (!topics.isEmpty()) { + topics.keySet().forEach(admin::createTopicIfNotExists); + List toSubscribe = new ArrayList<>(); + topics.forEach((topic, kafkaPartitions) -> { + if (kafkaPartitions == null) { + toSubscribe.add(topic); + } else { + consumer.assign(kafkaPartitions.stream() + .map(partition -> new TopicPartition(topic, partition)) + .toList()); + partitionsAssigned = true; + onPartitionsAssigned(); + } + }); + if (!toSubscribe.isEmpty()) { + consumer.subscribe(toSubscribe); + } + if (readFromBeginning) { + consumer.seekToBeginning(Collections.emptySet()); // for all assigned partitions + } } else { log.info("unsubscribe due to empty topic list"); consumer.unsubscribe(); @@ -88,6 +133,13 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue log.trace("poll topic {} maxDuration {}", getTopic(), durationInMillis); ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (!partitionsAssigned) { + if (readFromBeginning) { + consumer.seekToBeginning(Collections.emptySet()); + } + partitionsAssigned = true; + onPartitionsAssigned(); + } stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); @@ -96,11 +148,36 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue return Collections.emptyList(); } else { List> recordList = new ArrayList<>(256); - records.forEach(recordList::add); + records.forEach(record -> { + recordList.add(record); + if (stopWhenRead) { + int partition = record.partition(); + Long endOffset = endOffsets.get(partition); + if (endOffset == null) { + log.warn("End offset not found for {} [{}]", record.topic(), partition); + return; + } + log.trace("[{}-{}] Got record offset {}, expected end offset: {}", record.topic(), partition, record.offset(), endOffset - 1); + if (record.offset() >= endOffset - 1) { + endOffsets.remove(partition); + } + } + }); + if (endOffsets != null && endOffsets.isEmpty()) { + log.info("Reached end offsets for {}, stopping consumer", consumer.assignment()); + stop(); + } return recordList; } } + private void onPartitionsAssigned() { + if (stopWhenRead) { + endOffsets = consumer.endOffsets(consumer.assignment()).entrySet().stream() + .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); + } + } + @Override public T decode(ConsumerRecord record) throws IOException { return decoder.decode(new KafkaTbQueueMsg(record)); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index 3c9b85e925..6acf24b3f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -54,7 +54,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro private final TbQueueAdmin admin; - private final Set topics; + private final Set topics; @Getter private final String clientId; @@ -97,16 +97,21 @@ public class TbKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + send(tpi, msg.getKey().toString(), msg, callback); + } + + public void send(TopicPartitionInfo tpi, String key, T msg, TbQueueCallback callback) { try { - createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); + String topic = tpi.getFullTopicName(); + createTopicIfNotExist(topic); byte[] data = msg.getData(); ProducerRecord record; List
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); if (log.isDebugEnabled()) { addAnalyticHeaders(headers); } - record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); + Integer partition = tpi.isUseInternalPartition() ? tpi.getPartition().orElse(null) : null; + record = new ProducerRecord<>(topic, partition, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { if (callback != null) { @@ -130,12 +135,12 @@ public class TbKafkaProducerTemplate implements TbQueuePro } } - private void createTopicIfNotExist(TopicPartitionInfo tpi) { - if (topics.contains(tpi)) { + private void createTopicIfNotExist(String topic) { + if (topics.contains(topic)) { return; } - admin.createTopicIfNotExists(tpi.getFullTopicName()); - topics.add(tpi); + admin.createTopicIfNotExists(topic); + topics.add(topic); } @Override @@ -144,4 +149,5 @@ public class TbKafkaProducerTemplate implements TbQueuePro producer.close(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index ee529e8a68..2c4ccae003 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,12 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.edqs-events:}") + private String edqsEventsProperties; + @Value("${queue.kafka.topic-properties.edqs-requests:}") + private String edqsRequestsProperties; + @Value("${queue.kafka.topic-properties.edqs-state:}") + private String edqsStateProperties; @Getter private Map coreConfigs; @@ -79,6 +85,12 @@ public class TbKafkaTopicConfigs { private Map edgeConfigs; @Getter private Map edgeEventConfigs; + @Getter + private Map edqsEventsConfigs; + @Getter + private Map edqsRequestsConfigs; + @Getter + private Map edqsStateConfigs; @PostConstruct private void init() { @@ -97,6 +109,9 @@ public class TbKafkaTopicConfigs { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + edqsEventsConfigs = PropertyUtils.getProps(edqsEventsProperties); + edqsRequestsConfigs = PropertyUtils.getProps(edqsRequestsProperties); + edqsStateConfigs = PropertyUtils.getProps(edqsStateProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java new file mode 100644 index 0000000000..094f22c423 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; + +public interface EdqsClientQueueFactory { + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index d70cad159b..a7c351a3c9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; @@ -23,13 +24,19 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -43,37 +50,21 @@ import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +@RequiredArgsConstructor public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final TopicService topicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin queueAdmin; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final EdqsConfig edqsConfig; private final InMemoryStorage storage; - public InMemoryMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueVersionControlSettings vcSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - InMemoryStorage storage) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - this.storage = storage; - } - @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); @@ -209,6 +200,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + TbQueueProducer> requestProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueConsumer> responseConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getResponsesTopic()); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(queueAdmin) + .requestTemplate(requestProducer) + .responseTemplate(responseConsumer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dd5d61e834..7a83264dfc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,11 +25,13 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -48,6 +50,8 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -80,6 +84,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueVersionControlSettings vcSettings; private final TbQueueEdgeSettings edgeSettings; private final TbKafkaConsumerStatsService consumerStatsService; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -94,6 +99,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -107,7 +114,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, - TbKafkaTopicConfigs kafkaTopicConfigs) { + TbKafkaTopicConfigs kafkaTopicConfigs, + EdqsConfig edqsConfig) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -119,6 +127,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +142,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -490,6 +501,42 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .groupId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -523,4 +570,5 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi edgeAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index cc0e044917..2ac607a346 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,11 +24,13 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -47,6 +49,8 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -79,6 +83,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -93,6 +98,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -107,6 +114,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, + EdqsConfig edqsConfig, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -119,6 +127,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +142,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -439,6 +450,42 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .groupId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -469,4 +516,5 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { vcAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 87a1a69c2e..b681f1fc2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,11 +23,13 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -43,6 +45,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -81,6 +84,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin edqsEventsAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -111,6 +115,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); } @Override @@ -293,6 +298,20 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + throw new UnsupportedOperationException(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index c4002f4d3e..ec47177f28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -42,7 +42,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index ec31763baa..913bd99ea7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -76,7 +76,7 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbUsageStatsMsgProducer(); - /** + /** * Used to push messages to other instances of TB Core Service * * @return diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index c406aeb311..d58441c0ad 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -36,7 +36,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index f00cc2c103..5b120e42e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -43,9 +43,8 @@ public class PropertyUtils { } public static Map getProps(Map defaultProperties, String propertiesStr, Function> parser) { - Map properties = defaultProperties; + Map properties = new HashMap<>(defaultProperties); if (StringUtils.isNotBlank(propertiesStr)) { - properties = new HashMap<>(properties); properties.putAll(parser.apply(propertiesStr)); } return properties; diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index b7a3ae20d3..6e8c9640e3 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -145,19 +145,19 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenMessages_whenSend_thenOK() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); final int msgCount = 10; for (int i = 0; i < msgCount; i++) { inst.send(getRequestMsgMock()); } assertThat(inst.pendingRequests.mappingCount(), equalTo((long) msgCount)); - verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @Test public void givenMessagesOverMaxPendingRequests_whenSend_thenImmediateFailedFutureForTheOfRequests() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); int msgOverflowCount = 10; for (int i = 0; i < inst.maxPendingRequests; i++) { @@ -167,7 +167,7 @@ public class DefaultTbQueueRequestTemplateTest { assertThat("max pending requests overflow", inst.send(getRequestMsgMock()).isDone(), is(true)); //overflow, immediate failed future } assertThat(inst.pendingRequests.mappingCount(), equalTo(inst.maxPendingRequests)); - verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @SuppressWarnings("unchecked") diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java new file mode 100644 index 0000000000..7847860dcd --- /dev/null +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.stats; + +public class DummyMessagesStats implements MessagesStats { + @Override + public void incrementTotal(int amount) { + + } + + @Override + public void incrementSuccessful(int amount) { + + } + + @Override + public void incrementFailed(int amount) { + + } + + @Override + public int getTotal() { + return 0; + } + + @Override + public int getSuccessful() { + return 0; + } + + @Override + public int getFailed() { + return 0; + } + + @Override + public void reset() { + + } + +} diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index 3833155c05..2217be8d23 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -22,7 +22,8 @@ public enum StatsType { JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"), HOUSEKEEPER("housekeeper"), - EDGE("edge"); + EDGE("edge"), + EDQS("edqs"); private final String name; diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index fe0e019537..8dc98377d3 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -559,6 +559,10 @@ public class JacksonUtil { } } + public static JsonNode getValueByPath(ObjectNode node, String path) { + return node.at("/" + path.replace(".", "/")); + } + @Data public static class JsonNodeProcessingTask { private final String path; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index ca0636761d..bde497ed94 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -175,7 +175,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } } } - consumer.subscribe(event.getPartitionsMap().values().stream().findAny().orElse(Collections.emptySet())); + consumer.subscribe(event.getNewPartitions().values().stream().findAny().orElse(Collections.emptySet())); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index e912872496..79a4ced3b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -17,7 +17,11 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.util.TbPair; import java.util.Collection; import java.util.List; @@ -45,6 +49,12 @@ public interface Dao { List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit); - default EntityType getEntityType() { return null; } + default PageData findAllFields(PageLink pageLink) { + throw new UnsupportedOperationException(); + } + + default EntityType getEntityType() { + return null; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 01a72ea58c..a4bf25cd4d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.dao.model.ToData; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -85,6 +86,10 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } + public static Pageable toPageable(PageLink pageLink, String... sortColumns) { + return toPageable(pageLink, Collections.emptyMap(), Arrays.stream(sortColumns).map(column -> new SortOrder(column, SortOrder.Direction.ASC)).toList(), false); + } + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return toPageable(pageLink, columnMap, sortOrders, true); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index d0c54550ba..19fe45ed88 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -15,9 +15,23 @@ */ package org.thingsboard.server.dao; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; -public interface TenantEntityDao { +public interface TenantEntityDao { + + default Long countByTenantId(TenantId tenantId) { + throw new UnsupportedOperationException(); + } + + default PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + throw new UnsupportedOperationException(); + } + + default ObjectType getType() { + throw new UnsupportedOperationException(); + } - Long countByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 42ab2e2545..ffc2ebe796 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -36,7 +36,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. @@ -226,4 +226,5 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD PageData findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink); PageData> getAllAssetTypes(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 7d4d5300f9..305bba4004 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -148,12 +148,11 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope); + PageData findAll(PageLink pageLink); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute); List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 0694178540..512ac5913d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; @@ -24,13 +26,18 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.Validator; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -45,16 +52,15 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true) @Primary @Slf4j +@RequiredArgsConstructor public class BaseAttributesService implements AttributesService { + private final AttributesDao attributesDao; + private final EdqsService edqsService; @Value("${sql.attributes.value_no_xss_validation:false}") private boolean valueNoXssValidation; - public BaseAttributesService(AttributesDao attributesDao) { - this.attributesDao = attributesDao; - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -98,26 +104,51 @@ public class BaseAttributesService implements AttributesService { public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, attribute); } @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> doSave(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + return Futures.transform(future, version -> { + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); + return version; + }, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); + List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); + return Futures.transform(Futures.allAsList(futures), result -> { + List keys = new ArrayList<>(); + for (TbPair keyVersionPair : result) { + String key = keyVersionPair.getFirst(); + Long version = keyVersionPair.getSecond(); + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + keys.add(key); + } + return keys; + }, MoreExecutors.directExecutor()); } @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + deleted.forEach(attribute -> { + AttributeScope scope = attribute.getKey(); + String key = attribute.getValue(); + if (scope != null && key != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); + } + }); return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 1ebd5c0ba4..c5ac50c9f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -24,18 +24,22 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; @@ -67,6 +71,7 @@ public class CachedAttributesService implements AttributesService { private final AttributesDao attributesDao; private final JpaExecutorService jpaExecutorService; private final CacheExecutorService cacheExecutorService; + private final EdqsService edqsService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; private final VersionedTbCache cache; @@ -79,11 +84,12 @@ public class CachedAttributesService implements AttributesService { public CachedAttributesService(AttributesDao attributesDao, JpaExecutorService jpaExecutorService, - StatsFactory statsFactory, + @Lazy EdqsService edqsService, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, VersionedTbCache cache) { this.attributesDao = attributesDao; this.jpaExecutorService = jpaExecutorService; + this.edqsService = edqsService; this.cacheExecutorService = cacheExecutorService; this.cache = cache; @@ -237,8 +243,10 @@ public class CachedAttributesService implements AttributesService { private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - put(entityId, scope, new BaseAttributeKvEntry(((BaseAttributeKvEntry)attribute).getKv(), attribute.getLastUpdateTs(), version)); + return Futures.transform(future, version -> { + BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); + put(entityId, scope, attributeKvEntry); + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); return version; }, cacheExecutor); } @@ -256,7 +264,9 @@ public class CachedAttributesService implements AttributesService { List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, keyVersionPair -> { String key = keyVersionPair.getFirst(); - cache.evict(new AttributeCacheKey(scope, entityId, key), keyVersionPair.getSecond()); + Long version = keyVersionPair.getSecond(); + cache.evict(new AttributeCacheKey(scope, entityId, key), version); + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); return key; }, cacheExecutor)).collect(Collectors.toList())); } @@ -269,6 +279,8 @@ public class CachedAttributesService implements AttributesService { String key = deleted.getValue(); if (scope != null && key != null) { cache.evict(new AttributeCacheKey(scope, entityId, key)); + // FIXME: version is Long.MAX_VALUE because we expect that the entity is deleted and there won't be any attributes after this + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); } }); return result.size(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index 7888161cd0..da050bdbc6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index 2878fbd8e3..bc6adbe8b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index 2e9e37577c..127abb5c3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -172,7 +172,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb var saved = dashboardDao.save(tenantId, dashboard); publishEvictEvent(new DashboardTitleEvictEvent(saved.getId())); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId) - .entityId(saved.getId()).created(dashboard.getId() == null).build()); + .entityId(saved.getId()).entity(saved).created(dashboard.getId() == null).build()); if (dashboard.getId() == null) { countService.publishCountEntityEvictEvent(tenantId, EntityType.DASHBOARD); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 2305e4419c..895eba06a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -39,7 +39,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 09e2be01e0..b444ef16ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -180,13 +180,12 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService findAll(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java index d813d2e3c9..af79d0469a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java @@ -35,7 +35,7 @@ import java.util.UUID; * The Interface EdgeDao. * */ -public interface EdgeDao extends Dao, TenantEntityDao { +public interface EdgeDao extends Dao, TenantEntityDao { Edge save(TenantId tenantId, Edge edge); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3cdcf5e874..9fcb6c92ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.HasLabel; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTitle; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -40,9 +42,21 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.sql.alarm.AlarmRepository; +import org.thingsboard.server.dao.sql.query.EntityMapping; +import org.thingsboard.server.dao.user.UserService; import java.util.ArrayList; import java.util.Collections; @@ -50,9 +64,12 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_GROUP_NAME; +import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_TYPE; import static org.thingsboard.server.common.data.id.EntityId.NULL_UUID; import static org.thingsboard.server.dao.service.Validator.validateEntityDataPageLink; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -79,12 +96,24 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe @Lazy EntityServiceRegistry entityServiceRegistry; + @Autowired @Lazy + private EdqsService edqsService; + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); + + if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + EdqsRequest request = EdqsRequest.builder() + .entityCountQuery(query) + .userPermissions(userPermissions) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityCountQueryResult(); + } return this.entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } @@ -95,6 +124,15 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); + if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + EdqsRequest request = EdqsRequest.builder() + .entityDataQuery(query) + .userPermissions(userPermissions) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityDataQueryResult(); + } + if (!isValidForOptimization(query)) { return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); } @@ -110,6 +148,25 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return new PageData<>(result, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); } + private boolean validForEdqs(EntityCountQuery query) { + return !(query.getEntityFilter() instanceof StateEntityOwnerFilter filter) || !EntityType.ALARM.equals(filter.getSingleEntity().getEntityType()); + } + + private EdqsResponse processEdqsRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + EdqsResponse response; + try { + log.info("Sending request to EDQS: {}", request); + response = edqsService.processRequest(tenantId, customerId, request).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + log.info("Received response from EDQS: {}", response); + if (response.getError() != null) { + throw new RuntimeException(response.getError()); + } + return response; + } + @Override public Optional fetchEntityName(TenantId tenantId, EntityId entityId) { log.trace("Executing fetchEntityName [{}]", entityId); @@ -184,6 +241,10 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe throw new IncorrectParameterException("Query entity filter type must be specified."); } else if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) { validateRelationQuery((RelationsQueryFilter) query.getEntityFilter()); + } else if (query.getEntityFilter().getType().equals(ENTITY_TYPE)) { + validateEntityTypeQuery((EntityTypeFilter) query.getEntityFilter()); + } else if (query.getEntityFilter().getType().equals(ENTITY_GROUP_NAME)) { + validateGroupNameQuery((EntityGroupNameFilter) query.getEntityFilter()); } } @@ -192,6 +253,18 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateEntityDataPageLink(query.getPageLink()); } + private static void validateEntityTypeQuery(EntityTypeFilter filter) { + if (filter.getEntityType() == null) { + throw new IncorrectParameterException("Entity type is required"); + } + } + + private static void validateGroupNameQuery(EntityGroupNameFilter filter) { + if (filter.getGroupType() == null) { + throw new IncorrectParameterException("Group type is required"); + } + } + private static void validateRelationQuery(RelationsQueryFilter queryFilter) { if (queryFilter.isMultiRoot() && queryFilter.getMultiRootEntitiesType() == null) { throw new IncorrectParameterException("Multi-root relation query filter should contain 'multiRootEntitiesType'"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 7b2f2d42c1..24f8aa4d77 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -18,7 +18,9 @@ package org.thingsboard.server.dao.entity; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.EnumMap; import java.util.List; @@ -26,26 +28,39 @@ import java.util.Map; @Service @Slf4j +@SuppressWarnings({"unchecked"}) public class EntityDaoRegistry { - private final Map> daos = new EnumMap<>(EntityType.class); + private final Map> tenantEntityDaos = new EnumMap<>(ObjectType.class); + private final Map> entityDaos = new EnumMap<>(EntityType.class); - private EntityDaoRegistry(List> daos) { - daos.forEach(dao -> { - EntityType entityType = dao.getEntityType(); - if (entityType != null) { - this.daos.put(entityType, dao); + private EntityDaoRegistry(List> entityDaos, List> tenantEntityDaos) { + entityDaos.forEach(dao -> { + if (dao.getEntityType() != null) { + this.entityDaos.put(dao.getEntityType(), dao); + } + }); + tenantEntityDaos.forEach(dao -> { + if (dao.getType() != null) { + this.tenantEntityDaos.put(dao.getType(), dao); } }); } - @SuppressWarnings("unchecked") public Dao getDao(EntityType entityType) { - Dao dao = (Dao) daos.get(entityType); + Dao dao = (Dao) entityDaos.get(entityType); if (dao == null) { throw new IllegalArgumentException("Missing dao for entity type " + entityType); } return dao; } + public TenantEntityDao getTenantEntityDao(ObjectType objectType) { + TenantEntityDao dao = tenantEntityDaos.get(objectType); + if (dao == null) { + throw new IllegalArgumentException("Missing tenant entity dao for entity type " + objectType); + } + return (TenantEntityDao) dao; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 43b5e5cdfb..2dc3dab627 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -123,7 +123,7 @@ public class EntityViewServiceImpl extends CachedVersionedEntityService { private final T oldEntity; private final EntityId entityId; private final Boolean created; + private final Boolean broadcastEvent; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index d404c3128c..f762be0429 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -281,6 +281,8 @@ public class ModelConstants { public static final String ALARM_COMMENT_TYPE = "type"; public static final String ALARM_COMMENT_COMMENT = "comment"; + public static final String ALARM_TYPES_TABLE_NAME = "alarm_types"; + /** * Entity relation constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java new file mode 100644 index 0000000000..60f5232142 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.UUID; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AlarmTypeCompositeKey implements Serializable { + + private UUID tenantId; + private String type; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java new file mode 100644 index 0000000000..729d7a44f0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.IdClass; +import jakarta.persistence.Table; +import lombok.Data; +import org.thingsboard.server.common.data.alarm.AlarmType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.ToData; + +import java.util.UUID; + +@Data +@Entity +@Table(name = ModelConstants.ALARM_TYPES_TABLE_NAME) +@IdClass(AlarmTypeCompositeKey.class) +public class AlarmTypeEntity implements ToData { + + @Id + @Column(name = ModelConstants.TENANT_ID_PROPERTY, nullable = false) + private UUID tenantId; + + @Id + @Column(name = ModelConstants.ALARM_TYPE_PROPERTY, nullable = false) + private String type; + + public AlarmTypeEntity() {} + + public AlarmTypeEntity(AlarmType alarmType) { + setTenantId(alarmType.getTenantId().getId()); + setType(alarmType.getType()); + } + + public AlarmTypeEntity(UUID tenantId, String type) { + this.tenantId = tenantId; + this.type = type; + } + + @Override + public AlarmType toData() { + AlarmType alarmType = new AlarmType(); + alarmType.setTenantId(TenantId.fromUUID(tenantId)); + alarmType.setType(type); + return alarmType; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java index 23fe580fa3..05930fbe00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java @@ -26,6 +26,7 @@ import jakarta.persistence.SqlResultSetMapping; import jakarta.persistence.SqlResultSetMappings; import jakarta.persistence.Table; import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; @@ -91,4 +92,12 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity { this.strKey = strKey; this.version = version; } + + @Override + public TsKvEntry toData() { + TsKvEntry tsKvEntry = super.toData(); + tsKvEntry.setVersion(version); + return tsKvEntry; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java index 94126ebd1b..d92d7cfd0f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { PageData findByTenantIdAndPageLink(TenantId tenantId, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index e83650b3b5..742ad45ffa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,5 +21,7 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityWithDataDao; public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { + Long sumDataSizeByTenantId(TenantId tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index 69c6a7c418..efc9dab41b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -51,7 +53,10 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public QueueStats save(TenantId tenantId, QueueStats queueStats) { log.trace("Executing save [{}]", queueStats); queueStatsValidator.validate(queueStats, QueueStats::getTenantId); - return queueStatsDao.save(tenantId, queueStats); + QueueStats savedQueueStats = queueStatsDao.save(tenantId, queueStats); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedQueueStats.getTenantId()).entityId(savedQueueStats.getId()) + .entity(savedQueueStats).created(queueStats.getId() == null).build()); + return savedQueueStats; } @Override @@ -93,6 +98,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu @Override public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { queueStatsDao.removeById(tenantId, id.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index d668f51dbd..d29b57c5c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.relation; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -41,6 +43,8 @@ public interface RelationDao { List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); + PageData findAll(PageLink pageLink); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 0e51cf256a..58cfdfe02a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -124,10 +124,9 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC if (ruleChain.getId() == null) { entityCountService.publishCountEntityEvictEvent(ruleChain.getTenantId(), EntityType.RULE_CHAIN); } - if (publishSaveEvent) { - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedRuleChain.getTenantId()) - .entity(savedRuleChain).entityId(savedRuleChain.getId()).created(ruleChain.getId() == null).build()); - } + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedRuleChain.getTenantId()) + .entity(savedRuleChain).entityId(savedRuleChain.getId()).created(ruleChain.getId() == null) + .broadcastEvent(publishSaveEvent).build()); return savedRuleChain; } catch (Exception e) { checkConstraintViolation(e, "rule_chain_external_id_unq_key", "Rule Chain with such external id already exists!"); @@ -289,9 +288,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC relationService.saveRelations(tenantId, relations); } ruleChain = ruleChainDao.save(tenantId, ruleChain); - if (publishSaveEvent) { - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId).entity(ruleChain).entityId(ruleChain.getId()).build()); - } + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId).entity(ruleChain) + .entityId(ruleChain.getId()).broadcastEvent(publishSaveEvent).build()); return RuleChainUpdateResult.successful(updatedRuleNodes); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index a394255b2e..0a8d449afe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -31,7 +31,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java index d0faee3c9c..991147cedb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java @@ -35,5 +35,9 @@ public interface AlarmCommentRepository extends JpaRepository findAllByAlarmId(@Param("alarmId") UUID alarmId, - Pageable pageable); + Pageable pageable); + + @Query("SELECT c FROM AlarmCommentEntity c WHERE c.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 6083dac6cf..9f57ab6e52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -414,4 +414,6 @@ public interface AlarmRepository extends JpaRepository { @Param("alarmSeverities") List alarmSeverities, int limit); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java new file mode 100644 index 0000000000..b84542e3c3 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.AlarmTypeCompositeKey; +import org.thingsboard.server.dao.model.sql.AlarmTypeEntity; + +import java.util.UUID; + +public interface AlarmTypeRepository extends JpaRepository { + + Page findByTenantId(UUID tenantId, Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index ccf82e1b5b..41e6f3dcc5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.alarm; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -42,4 +44,6 @@ public interface EntityAlarmRepository extends JpaRepository findAllByEntityId(UUID entityId); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java index 3d366a5a56..9d74b9bf8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmCommentInfo; import org.thingsboard.server.common.data.id.AlarmId; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmCommentDao; import org.thingsboard.server.dao.model.sql.AlarmCommentEntity; import org.thingsboard.server.dao.sql.JpaPartitionedAbstractDao; @@ -44,7 +46,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ALARM_COMMENT_TABL @Component @SqlDao @RequiredArgsConstructor -public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao { +public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao, TenantEntityDao { private final SqlPartitioningRepository partitioningRepository; @Value("${sql.alarm_comments.partition_size:168}") private int partitionSizeInHours; @@ -76,6 +78,11 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmCommentRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override protected Class getEntityClass() { return AlarmCommentEntity.class; @@ -85,4 +92,10 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao getRepository() { return alarmCommentRepository; } + + @Override + public ObjectType getType() { + return ObjectType.ALARM_COMMENT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index e433dfbd86..7aea2adbec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -29,6 +29,7 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @@ -57,6 +58,7 @@ import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.OriginatorAlarmFilter; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AlarmEntity; @@ -84,7 +86,7 @@ import static org.thingsboard.server.dao.DaoUtil.toPageable; @Slf4j @Component @SqlDao -public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao { +public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao, TenantEntityDao { @Autowired private AlarmRepository alarmRepository; @@ -551,9 +553,19 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ALARM; } + @Override + public ObjectType getType() { + return ObjectType.ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java new file mode 100644 index 0000000000..532f06eb03 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.alarm.AlarmType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.util.SqlDao; + +@Component +@SqlDao +public class JpaAlarmTypeDao implements TenantEntityDao { + + @Autowired + private AlarmTypeRepository alarmTypeRepository; + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmTypeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "tenantId", "type"))); + } + + @Override + public ObjectType getType() { + return ObjectType.ALARM_TYPE; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java new file mode 100644 index 0000000000..6bbc59559a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.alarm.EntityAlarm; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.util.SqlDao; + +@Component +@SqlDao +public class JpaEntityAlarmDao implements TenantEntityDao { + + @Autowired + private EntityAlarmRepository entityAlarmRepository; + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(entityAlarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "entityId", "alarmId"))); + } + + @Override + public ObjectType getType() { + return ObjectType.ENTITY_ALARM; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java index 372201bc47..bd3f35cf76 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; @@ -81,4 +82,8 @@ public interface AssetProfileRepository extends JpaRepository findAllTenantAssetProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetProfileFields(a.id, a.createdTime, a.tenantId," + + "a.name, a.version, a.isDefault) FROM AssetProfileEntity a") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index bf4798fa4d..1be4fc6075 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; @@ -216,4 +217,9 @@ public interface AssetRepository extends JpaRepository, Expor @Query(value = "SELECT DISTINCT new org.thingsboard.server.common.data.util.TbPair(a.tenantId , a.type) FROM AssetEntity a") Page> getAllAssetTypes(Pageable pageable); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetFields(a.id, a.createdTime, a.tenantId, a.customerId," + + "a.name, a.version, a.type, a.label, a.assetProfileId, a.additionalInfo) FROM AssetEntity a") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 0ef4370f30..569c13f376 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -22,8 +22,10 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -267,9 +269,24 @@ public class JpaAssetDao extends JpaAbstractDao implements A .map(AssetId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; } + @Override + public ObjectType getType() { + return ObjectType.ASSET; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java index 6ed74a5aa9..eb2fb6a9ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java @@ -21,13 +21,16 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.asset.AssetProfileDao; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -37,7 +40,7 @@ import java.util.Optional; import java.util.UUID; @Component -public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao { +public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao, TenantEntityDao { @Autowired private AssetProfileRepository assetProfileRepository; @@ -138,9 +141,24 @@ public class JpaAssetProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(assetProfileRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ASSET_PROFILE; } + @Override + public ObjectType getType() { + return ObjectType.ASSET_PROFILE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index a0e1f1447d..b344a0ca41 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AttributeScope; @@ -30,6 +31,8 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; @@ -48,6 +51,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -152,6 +156,12 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return DaoUtil.convertDataList(Lists.newArrayList(attributes)); } + @Override + public PageData findAll(PageLink pageLink) { + Page attributes = attributeKvRepository.findAll(DaoUtil.toPageable(pageLink)); + return DaoUtil.pageToPageData(attributes); + } + @Override public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { if (deviceProfileId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 448b850ba7..19b3ec8fe8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -20,6 +20,8 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CustomerEntity; @@ -55,4 +57,8 @@ public interface CustomerRepository extends JpaRepository, nativeQuery = true) Page findCustomersWithTheSameTitle(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.CustomerFields(c.id, c.createdTime, c.tenantId, " + + "c.title, c.version, c.additionalInfo, c.country, c.state, c.city, c.address, c.address2, c.zip, c.phone, c.email) FROM CustomerEntity c") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 497a2917d8..a371803763 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -20,6 +20,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -104,9 +106,24 @@ public class JpaCustomerDao extends JpaAbstractDao imp ); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(customerRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; } + @Override + public ObjectType getType() { + return ObjectType.CUSTOMER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 6f182351c1..9932819ce0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -46,4 +47,8 @@ public interface DashboardRepository extends JpaRepository findAllIds(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DashboardFields(d.id, d.createdTime, d.tenantId, " + + "d.customerId, d.title, d.version) FROM DashboardEntity d") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 640107add0..436c32d32b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -20,6 +20,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -90,9 +92,24 @@ public class JpaDashboardDao extends JpaAbstractDao return DaoUtil.pageToPageData(dashboardRepository.findAllIds(DaoUtil.toPageable(pageLink)).map(DashboardId::new)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(dashboardRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; } + @Override + public ObjectType getType() { + return ObjectType.DASHBOARD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index c136d361e2..f2b541cb45 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -36,4 +38,8 @@ public interface DeviceCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index b51324f11a..e41796381d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -23,6 +23,8 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; +import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; @@ -92,4 +94,8 @@ public interface DeviceProfileRepository extends JpaRepository findAllTenantDeviceProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields(d.id, d.createdTime, d.tenantId," + + "d.name, d.version, d.type, d.isDefault) FROM DeviceProfileEntity d") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c55210b606..161c201404 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -194,4 +195,9 @@ public interface DeviceRepository extends JpaRepository, Exp @Query("SELECT externalId FROM DeviceEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceFields(d.id, d.createdTime, d.tenantId, d.customerId," + + "d.name, d.version, d.type, d.label, d.deviceProfileId, d.additionalInfo) FROM DeviceEntity d") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 7571c00b6a..31d2aa6cb9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -19,10 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceCredentialsDao; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -36,7 +40,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao { +public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao, TenantEntityDao { @Autowired private DeviceCredentialsRepository deviceCredentialsRepository; @@ -67,4 +71,14 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(deviceCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.DEVICE_CREDENTIALS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 48bb998016..2763e5dca7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,7 +29,9 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -282,9 +284,24 @@ public class JpaDeviceDao extends JpaAbstractDao implement .map(DeviceId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(deviceRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; } + @Override + public ObjectType getType() { + return ObjectType.DEVICE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 4e1594eeff..ff6e61c051 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -24,12 +24,15 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -41,7 +44,7 @@ import java.util.UUID; @Component @SqlDao -public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao { +public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao, TenantEntityDao { @Autowired private DeviceProfileRepository deviceProfileRepository; @@ -156,9 +159,24 @@ public class JpaDeviceProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findDeviceProfiles(tenantId, pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(deviceProfileRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; } + @Override + public ObjectType getType() { + return ObjectType.DEVICE_PROFILE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java index 1bcfab7ab3..e75ae5984a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.dao.model.sql.EdgeEntity; import org.thingsboard.server.dao.model.sql.EdgeInfoEntity; @@ -154,4 +155,8 @@ public interface EdgeRepository extends JpaRepository { EdgeEntity findByRoutingKey(String routingKey); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.EdgeFields(e.id, e.createdTime, e.tenantId, e.customerId," + + "e.name, e.version, e.type, e.label, e.additionalInfo) FROM EdgeEntity e") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 2249ced97c..eb46d5bec7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -22,8 +22,10 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -219,9 +221,24 @@ public class JpaEdgeDao extends JpaAbstractDao implements Edge return edgeRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findEdgesByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(edgeRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.EDGE; } + @Override + public ObjectType getType() { + return ObjectType.EDGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 8f2e61d787..b4ad0478e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -145,4 +146,8 @@ public interface EntityViewRepository extends JpaRepository findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 98c5ad7036..82e7eab609 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -24,11 +24,14 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -47,8 +50,7 @@ import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityTypesToDto; @Component @Slf4j @SqlDao -public class JpaEntityViewDao extends JpaAbstractDao - implements EntityViewDao { +public class JpaEntityViewDao extends JpaAbstractDao implements EntityViewDao, TenantEntityDao { @Autowired private EntityViewRepository entityViewRepository; @@ -218,8 +220,24 @@ public class JpaEntityViewDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(entityViewRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ENTITY_VIEW; } + + @Override + public ObjectType getType() { + return ObjectType.ENTITY_VIEW; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java index 48a7df0f3b..e6df0ff235 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.NotificationRuleId; import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.config.Notif import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.NotificationRuleEntity; import org.thingsboard.server.dao.model.sql.NotificationRuleInfoEntity; import org.thingsboard.server.dao.notification.NotificationRuleDao; @@ -41,7 +43,7 @@ import java.util.UUID; @Component @SqlDao @RequiredArgsConstructor -public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao { +public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao, TenantEntityDao { private final NotificationRuleRepository notificationRuleRepository; @@ -101,6 +103,11 @@ public class JpaNotificationRuleDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationRuleEntity.class; @@ -116,4 +123,9 @@ public class JpaNotificationRuleDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationTargetEntity.class; @@ -116,4 +122,9 @@ public class JpaNotificationTargetDao extends JpaAbstractDao implements NotificationTemplateDao { +public class JpaNotificationTemplateDao extends JpaAbstractDao implements NotificationTemplateDao, TenantEntityDao { private final NotificationTemplateRepository notificationTemplateRepository; @@ -83,6 +85,11 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected JpaRepository getRepository() { return notificationTemplateRepository; @@ -93,4 +100,9 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao implements OtaPackageDao { +public class JpaOtaPackageDao extends JpaAbstractDao implements OtaPackageDao, TenantEntityDao { @Autowired private OtaPackageRepository otaPackageRepository; @@ -52,9 +58,20 @@ public class JpaOtaPackageDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(otaPackageRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.OTA_PACKAGE; } + @Override + public ObjectType getType() { + return ObjectType.OTA_PACKAGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java index 707a6e7062..74f14a9697 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java @@ -93,4 +93,5 @@ public class JpaOtaPackageInfoDao extends JpaAbstractDao { + @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); + + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 05da22f9a5..6c8f48446e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -28,6 +28,10 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.Resource; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataPageLink; @@ -128,7 +132,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { public PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds) { return transactionTemplate.execute(trStatus -> { AlarmDataPageLink pageLink = query.getPageLink(); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); ctx.addUuidListParameter("entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); StringBuilder selectPart = new StringBuilder(FIELDS_SELECTION); StringBuilder fromPart = new StringBuilder(" from alarm_info a "); @@ -315,7 +319,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { @Override public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); if (query.isSearchPropagatedAlarms()) { ctx.append("select count(distinct(a.id)) from alarm_info a "); @@ -402,7 +406,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { }); } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && selectionMapping != null && !selectionMapping.isEmpty()) { String lowerSearchText = searchText.toLowerCase() + "%"; List searchPredicates = selectionMapping.stream() @@ -420,7 +424,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } } - private String buildPermissionsQuery(TenantId tenantId, QueryContext ctx) { + private String buildPermissionsQuery(TenantId tenantId, SqlQueryContext ctx) { StringBuilder permissionsQuery = new StringBuilder(); ctx.addUuidParameter("permissions_tenant_id", tenantId.getId()); permissionsQuery.append(" a.tenant_id = :permissions_tenant_id and ea.tenant_id = :permissions_tenant_id "); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 36ae779edd..31443f7f47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; import org.thingsboard.server.common.data.query.AssetTypeFilter; @@ -334,7 +335,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); if (query.getKeyFilters() == null || query.getKeyFilters().isEmpty()) { ctx.append("select count(e.id) from "); ctx.append(addEntityTableQuery(ctx, query.getEntityFilter())); @@ -416,7 +417,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck) { return transactionTemplate.execute(status -> { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, ignorePermissionCheck)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, ignorePermissionCheck)); EntityDataPageLink pageLink = query.getPageLink(); List mappings = EntityKeyMapping.prepareKeyMapping(entityType, query); @@ -524,7 +525,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { }); } - private String buildEntityWhere(QueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { + private String buildEntityWhere(SqlQueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { String permissionQuery = this.buildPermissionQuery(ctx, entityFilter); String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, entityFieldsFilters, entityFilter.getType()); @@ -538,7 +539,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return result; } - private String buildPermissionQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildPermissionQuery(SqlQueryContext ctx, EntityFilter entityFilter) { if (ctx.isIgnorePermissionCheck()) { return "1=1"; } @@ -575,7 +576,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String defaultPermissionQuery(QueryContext ctx) { + private String defaultPermissionQuery(SqlQueryContext ctx) { ctx.addUuidParameter("permissions_tenant_id", ctx.getTenantId().getId()); if (ctx.getCustomerId() != null && !ctx.getCustomerId().isNullUid()) { ctx.addUuidParameter("permissions_customer_id", ctx.getCustomerId().getId()); @@ -593,7 +594,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String buildEntityFilterQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildEntityFilterQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case SINGLE_ENTITY: return this.singleEntityQuery(ctx, (SingleEntityFilter) entityFilter); @@ -619,7 +620,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String addEntityTableQuery(QueryContext ctx, EntityFilter entityFilter) { + private String addEntityTableQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case RELATIONS_QUERY: return relationQuery(ctx, (RelationsQueryFilter) entityFilter); @@ -640,7 +641,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String entitySearchQuery(QueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { + private String entitySearchQuery(SqlQueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, additional_info " @@ -680,7 +681,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return query; } - private String relationQuery(QueryContext ctx, RelationsQueryFilter entityFilter) { + private String relationQuery(SqlQueryContext ctx, RelationsQueryFilter entityFilter) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = SELECT_TENANT_ID + ", " + SELECT_CUSTOMER_ID @@ -763,7 +764,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return "( " + selectFields + from + ")"; } - private String buildEtfCondition(QueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { + private String buildEtfCondition(SqlQueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { StringBuilder whereFilter = new StringBuilder(); String relationType = etf.getRelationType(); List entityTypes = etf.getEntityTypes(); @@ -812,7 +813,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return from; } - private String buildAliasWhereQuery(QueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { + private String buildAliasWhereQuery(SqlQueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { List aliasFiltersMapping = selectionMapping.stream().filter(mapping -> !mapping.isLatest() && mapping.getEntityKeyColumn() == null) .collect(Collectors.toList()); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, aliasFiltersMapping, entityFilter.getType()); @@ -822,12 +823,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { result += " where (" + entityFieldsQuery + ")"; } if (!searchTextQuery.isEmpty()) { - result += (result.isEmpty() ? " where ": " and ") + "(" + searchTextQuery + ") "; + result += (result.isEmpty() ? " where " : " and ") + "(" + searchTextQuery + ") "; } return result; } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && !selectionMapping.isEmpty()) { String sqlSearchText = "%" + searchText + "%"; ctx.addStringParameter("lowerSearchTextParam", sqlSearchText); @@ -844,17 +845,17 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String singleEntityQuery(QueryContext ctx, SingleEntityFilter filter) { + private String singleEntityQuery(SqlQueryContext ctx, SingleEntityFilter filter) { ctx.addUuidParameter("entity_filter_single_entity_id", filter.getSingleEntity().getId()); return "e.id=:entity_filter_single_entity_id"; } - private String entityListQuery(QueryContext ctx, EntityListFilter filter) { + private String entityListQuery(SqlQueryContext ctx, EntityListFilter filter) { ctx.addUuidListParameter("entity_filter_entity_ids", filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toList())); return "e.id in (:entity_filter_entity_ids)"; } - private String entityNameQuery(QueryContext ctx, EntityNameFilter filter) { + private String entityNameQuery(SqlQueryContext ctx, EntityNameFilter filter) { ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter()); String nameColumn = getNameColumn(filter.getEntityType()); if (filter.getEntityNameFilter().startsWith("%") || filter.getEntityNameFilter().endsWith("%")) { @@ -864,7 +865,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return String.format("e.%s ilike concat(:entity_filter_name_filter, '%%')", nameColumn); } - private String typeQuery(QueryContext ctx, EntityFilter filter) { + private String typeQuery(SqlQueryContext ctx, EntityFilter filter) { List types; String name; String nameColumn; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 643b4edeec..a6ab1bee7f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -37,7 +37,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { private long logQueriesThreshold; @Override - public void logQuery(QueryContext ctx, String query, long duration) { + public void logQuery(SqlQueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { String sqlToUse = substituteParametersInSqlString(query, ctx); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java new file mode 100644 index 0000000000..91934293a6 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import com.google.common.util.concurrent.ListenableFuture; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsService; + +@Service +@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "false", matchIfMissing = true) +public class DummyEdqsService implements EdqsService { + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isApiEnabled() { + return false; + } + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {} + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) {} + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) {} + + @Override + public void processSystemMsg(ToCoreEdqsMsg request) {} + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 599ac3d918..b02c208dae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -103,7 +103,7 @@ public class EntityKeyMapping { public static final List labeledEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, ADDITIONAL_INFO); public static final List contactBasedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, EMAIL, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO); - public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); + public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); public static final Set commonEntityFieldsSet = new HashSet<>(commonEntityFields); public static final Set relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO, RELATED_PARENT_ID)); @@ -265,7 +265,7 @@ public class EntityKeyMapping { return alias; } - public Stream toQueries(QueryContext ctx, EntityFilterType filterType) { + public Stream toQueries(SqlQueryContext ctx, EntityFilterType filterType) { if (hasFilter()) { String keyAlias = (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD) && getEntityKeyColumn() != null) ? "e" : alias; return keyFilters.stream().map(keyFilter -> @@ -275,7 +275,7 @@ public class EntityKeyMapping { } } - public String toLatestJoin(QueryContext ctx, EntityFilter entityFilter, EntityType entityType) { + public String toLatestJoin(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType) { String entityTypeStr; if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) { entityTypeStr = "entities.entity_type"; @@ -303,9 +303,9 @@ public class EntityKeyMapping { if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { scope = AttributeScope.CLIENT_SCOPE.getId(); } else if (entityKey.getType().equals(EntityKeyType.SHARED_ATTRIBUTE)) { - scope = AttributeScope.SHARED_SCOPE.getId();; + scope = AttributeScope.SHARED_SCOPE.getId(); ; } else { - scope = AttributeScope.SERVER_SCOPE.getId();; + scope = AttributeScope.SERVER_SCOPE.getId(); ; } query = String.format("%s AND %s.attribute_type=%s %s", query, alias, scope, filterQuery); } else { @@ -318,7 +318,7 @@ public class EntityKeyMapping { } } - private boolean hasFilterValues(QueryContext ctx) { + private boolean hasFilterValues(SqlQueryContext ctx) { return Arrays.stream(ctx.getParameterNames()).anyMatch(parameterName -> { return !parameterName.equals(getKeyId()) && parameterName.startsWith(alias); }); @@ -333,14 +333,14 @@ public class EntityKeyMapping { Collectors.joining(", ")); } - public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { + public static String buildLatestJoins(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { return latestMappings.stream() .filter(mapping -> !countQuery || mapping.hasFilter()) .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)) .collect(Collectors.joining(" ")); } - public static String buildQuery(QueryContext ctx, List mappings, EntityFilterType filterType) { + public static String buildQuery(SqlQueryContext ctx, List mappings, EntityFilterType filterType) { return mappings.stream() .flatMap(mapping -> mapping.toQueries(ctx, filterType)) .filter(StringUtils::isNotEmpty) @@ -510,12 +510,12 @@ public class EntityKeyMapping { return getValueAlias() + "_so_num"; } - private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter, + private String buildKeyQuery(SqlQueryContext ctx, String alias, KeyFilter keyFilter, EntityFilterType filterType) { return this.buildPredicateQuery(ctx, alias, keyFilter.getKey(), keyFilter.getPredicate(), filterType); } - private String buildPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (predicate.getType().equals(FilterPredicateType.COMPLEX)) { return this.buildComplexPredicateQuery(ctx, alias, key, (ComplexFilterPredicate) predicate, filterType); @@ -524,7 +524,7 @@ public class EntityKeyMapping { } } - private String buildComplexPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildComplexPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, ComplexFilterPredicate predicate, EntityFilterType filterType) { String result = predicate.getPredicates().stream() .map(keyFilterPredicate -> this.buildPredicateQuery(ctx, alias, key, keyFilterPredicate, filterType)) @@ -536,7 +536,7 @@ public class EntityKeyMapping { return result; } - private String buildSimplePredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildSimplePredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (key.getType().equals(EntityKeyType.ENTITY_FIELD)) { String field = (getEntityKeyColumn() != null) ? alias + "." + getEntityKeyColumn() : alias; @@ -571,7 +571,7 @@ public class EntityKeyMapping { } } - private String buildStringPredicateQuery(QueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { + private String buildStringPredicateQuery(SqlQueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { String operationField = field; String paramName = getNextParameterName(field); String value = stringFilterPredicate.getValue().getValue(); @@ -624,7 +624,7 @@ public class EntityKeyMapping { return String.format("((%s is not null and %s)", field, stringOperationQuery); } - private String buildNumericPredicateQuery(QueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { + private String buildNumericPredicateQuery(SqlQueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { String paramName = getNextParameterName(field); ctx.addDoubleParameter(paramName, numericFilterPredicate.getValue().getValue()); String numericOperationQuery = ""; @@ -651,7 +651,7 @@ public class EntityKeyMapping { return String.format("(%s is not null and %s)", field, numericOperationQuery); } - private String buildBooleanPredicateQuery(QueryContext ctx, String field, + private String buildBooleanPredicateQuery(SqlQueryContext ctx, String field, BooleanFilterPredicate booleanFilterPredicate) { String paramName = getNextParameterName(field); ctx.addBooleanParameter(paramName, booleanFilterPredicate.getValue().getValue()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java index f455e2d591..cbb3d4141d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java @@ -17,5 +17,5 @@ package org.thingsboard.server.dao.sql.query; public interface QueryLogComponent { - void logQuery(QueryContext ctx, String query, long duration); + void logQuery(SqlQueryContext ctx, String query, long duration); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java index 6869703bce..42ac8280ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java @@ -21,6 +21,7 @@ import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; import java.sql.Types; import java.util.HashMap; @@ -29,14 +30,14 @@ import java.util.Map; import java.util.UUID; @Slf4j -public class QueryContext implements SqlParameterSource { +public class SqlQueryContext implements SqlParameterSource { private static final UUIDJdbcType UUID_TYPE = UUIDJdbcType.INSTANCE; - private final QuerySecurityContext securityCtx; + private final QueryContext securityCtx; private final StringBuilder query; private final Map params; - public QueryContext(QuerySecurityContext securityCtx) { + public SqlQueryContext(QueryContext securityCtx) { this.securityCtx = securityCtx; query = new StringBuilder(); params = new HashMap<>(); @@ -48,7 +49,7 @@ public class QueryContext implements SqlParameterSource { if (oldParam != null && oldParam.value != null && !oldParam.value.equals(newParam.value)) { throw new RuntimeException("Parameter with name: " + name + " was already registered!"); } - if(value == null){ + if (value == null) { log.warn("[{}][{}][{}] Trying to set null value", getTenantId(), getCustomerId(), name); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java index e4563b7d35..c84392842a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java @@ -22,11 +22,13 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.QueueEntity; import org.thingsboard.server.dao.queue.QueueDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +40,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaQueueDao extends JpaAbstractDao implements QueueDao { +public class JpaQueueDao extends JpaAbstractDao implements QueueDao, TenantEntityDao { @Autowired private QueueRepository queueRepository; @@ -87,9 +89,19 @@ public class JpaQueueDao extends JpaAbstractDao implements Q .findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findQueuesByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE; } + @Override + public ObjectType getType() { + return ObjectType.QUEUE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index f680a42dc1..b78bbb8f37 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -74,6 +75,11 @@ public class JpaQueueStatsDao extends JpaAbstractDao findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(queueStatsRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE_STATS; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java index 585f010469..594e80e42a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.dao.model.sql.QueueStatsEntity; import java.util.List; @@ -45,4 +46,8 @@ public interface QueueStatsRepository extends JpaRepository findByTenantIdAndIdIn(UUID tenantId, List queueStatsIds); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.QueueStatsFields(q.id, q.createdTime," + + "q.tenantId, q.queueName, q.serviceId) FROM QueueStatsEntity q") + Page findAllFields(Pageable pageable); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 0e726f8917..1df7115518 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -24,6 +24,8 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -127,6 +129,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public PageData findAll(PageLink pageLink) { + return DaoUtil.toPageData(relationRepository.findAll(DaoUtil.toPageable(pageLink))); + } + @Override public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { return service.submit(() -> checkRelation(tenantId, from, to, relationType, typeGroup)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 0f56a413df..d945f1f640 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.relation; +import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -84,4 +85,6 @@ public interface RelationRepository @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType and r.relationTypeGroup in :relationTypeGroups") void deleteByFromIdAndFromTypeAndRelationTypeGroupIn(@Param("fromId") UUID fromId, @Param("fromType") String fromType, @Param("relationTypeGroups") List relationTypeGroups); + @Query("SELECT e FROM RelationEntity e ORDER BY e.fromId, e.fromType, e.toId, e.toType, e.relationType, e.relationTypeGroup") + Page findAll(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java index f8927aaa81..ca0045b153 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.TbResourceEntity; import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +40,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao { +public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao, TenantEntityDao { private final TbResourceRepository resourceRepository; @@ -145,4 +147,9 @@ public class JpaTbResourceDao extends JpaAbstractDao implements RpcDao { +public class JpaRpcDao extends JpaAbstractDao implements RpcDao, TenantEntityDao { private final RpcRepository rpcRepository; @@ -74,9 +76,19 @@ public class JpaRpcDao extends JpaAbstractDao implements RpcDao return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findAllRpcByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.RPC; } + @Override + public ObjectType getType() { + return ObjectType.RPC; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index eaf151f256..588be25ba9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -20,6 +20,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -133,9 +135,24 @@ public class JpaRuleChainDao extends JpaAbstractDao return findRootRuleChainByTenantIdAndType(tenantId, RuleChainType.CORE); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(ruleChainRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; } + @Override + public ObjectType getType() { + return ObjectType.RULE_CHAIN; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index c267eab4a8..307f15c7f4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import org.thingsboard.server.dao.rule.RuleNodeDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -40,7 +42,7 @@ import java.util.stream.Collectors; @Slf4j @Component @SqlDao -public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao { +public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao, TenantEntityDao { @Autowired private RuleNodeRepository ruleNodeRepository; @@ -106,9 +108,19 @@ public class JpaRuleNodeDao extends JpaAbstractDao imp ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(ruleNodeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.RULE_NODE; } + @Override + public ObjectType getType() { + return ObjectType.RULE_NODE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 305d5993d4..6713e4b645 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.RuleChainEntity; @@ -70,4 +71,7 @@ public interface RuleChainRepository extends JpaRepository findAllFields(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index 264ea88326..bf19d61a9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -63,4 +63,7 @@ public interface RuleNodeRepository extends JpaRepository @Query("DELETE FROM RuleNodeEntity e where e.id in :ids") void deleteByIdIn(@Param("ids") List ids); + @Query("SELECT n FROM RuleNodeEntity n WHERE n.ruleChainId IN (SELECT rc.id FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index b390bfffd8..78cf891870 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.settings; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; @@ -33,4 +35,6 @@ public interface AdminSettingsRepository extends JpaRepository findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 9aa348876e..9e0f6a70cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -21,7 +21,12 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -32,7 +37,7 @@ import java.util.UUID; @Component @SqlDao @Slf4j -public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao { +public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao, TenantEntityDao { @Autowired private AdminSettingsRepository adminSettingsRepository; @@ -68,4 +73,14 @@ public class JpaAdminSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(adminSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.ADMIN_SETTINGS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index cea414a42f..ff014d012f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; @@ -94,4 +95,15 @@ public class JpaTenantDao extends JpaAbstractDao implement .map(TenantId::fromUUID) .collect(Collectors.toList()); } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(tenantRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java index 372c5bc9b8..4668c44b61 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -87,6 +88,11 @@ public class JpaTenantProfileDao extends JpaAbstractDao findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(tenantProfileRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.TENANT_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java index a9cbf3d591..b7a57d0942 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.dao.model.sql.TenantProfileEntity; import java.util.List; @@ -55,4 +56,8 @@ public interface TenantProfileRepository extends JpaRepository findByIdIn(List ids); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantProfileFields(t.id, t.createdTime, t.name," + + "t.isDefault) FROM TenantProfileEntity t") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 795404399e..86dbeaff80 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; @@ -53,4 +54,8 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantFields(t.id, t.createdTime, t.title, t.version," + + "t.additionalInfo, t.country, t.state, t.city, t.address, t.address2, t.zip, t.phone, t.email, t.region) FROM TenantEntity t") + Page findAllFields(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index 88873e8671..509796879c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -15,11 +15,14 @@ */ package org.thingsboard.server.dao.sql.usagerecord; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import java.util.UUID; @@ -35,6 +38,8 @@ public interface ApiUsageStateRepository extends JpaRepository findAllByTenantId(UUID tenantId, Pageable pageable); + @Transactional @Modifying @Query("DELETE FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId") @@ -44,4 +49,9 @@ public interface ApiUsageStateRepository extends JpaRepository findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java index 63c0d9d3d1..85d9d18839 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -19,8 +19,12 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -72,9 +76,24 @@ public class JpaApiUsageStateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(apiUsageStateRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(apiUsageStateRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.API_USAGE_STATE; } + @Override + public ObjectType getType() { + return ObjectType.API_USAGE_STATE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java index 316e34de87..171d10b0c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java @@ -18,9 +18,15 @@ package org.thingsboard.server.dao.sql.user; import lombok.RequiredArgsConstructor; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserAuthSettings; +import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoFaSettings; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserAuthSettingsDao; @@ -31,7 +37,7 @@ import java.util.UUID; @Component @RequiredArgsConstructor @SqlDao -public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao { +public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao, TenantEntityDao { private final UserAuthSettingsRepository repository; @@ -45,6 +51,18 @@ public class JpaUserAuthSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + PageData data = DaoUtil.toPageData(repository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + data.getData().forEach(settings -> { + AccountTwoFaSettings twoFaSettings = settings.getTwoFaSettings(); + if (twoFaSettings != null && twoFaSettings.getConfigs() != null) { + twoFaSettings.getConfigs().values().forEach(config -> config.setSerializeHiddenFields(true)); + } + }); + return data; + } + @Override protected Class getEntityClass() { return UserAuthSettingsEntity.class; @@ -55,4 +73,9 @@ public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserCredentialsDao { +public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao, TenantEntityDao { @Autowired private UserCredentialsRepository userCredentialsRepository; @@ -84,4 +88,14 @@ public class JpaUserCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.USER_CREDENTIALS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 7749dbff29..55d74634d6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -19,7 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; @@ -139,9 +141,24 @@ public class JpaUserDao extends JpaAbstractDao implements User return userRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(userRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.USER; } + @Override + public ObjectType getType() { + return ObjectType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index 671304256e..b41bfa7728 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -18,14 +18,17 @@ package org.thingsboard.server.dao.sql.user; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.common.data.settings.UserSettingsType; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserSettingsEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.user.UserSettingsDao; import org.thingsboard.server.dao.util.SqlDao; @@ -34,7 +37,7 @@ import java.util.List; @Slf4j @Component @SqlDao -public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService implements UserSettingsDao { +public class JpaUserSettingsDao implements UserSettingsDao, TenantEntityDao { @Autowired private UserSettingsRepository userSettingsRepository; @@ -66,4 +69,14 @@ public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService i return DaoUtil.convertDataList(userSettingsRepository.findByTypeAndPathExisting(type.name(), path)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.USER_SETTINGS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java index cea5d428dd..3c9d69f085 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -35,4 +37,7 @@ public interface UserAuthSettingsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java index 4aca40647c..682219ed3a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; @@ -52,4 +55,7 @@ public interface UserCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 79b0a00308..e67330a1a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.sql.UserEntity; @@ -71,4 +72,9 @@ public interface UserRepository extends JpaRepository { Long countByTenantId(UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.UserFields(u.id, u.createdTime, u.tenantId," + + "u.customerId, u.version, u.firstName, u.lastName, u.email, u.phone, u.additionalInfo) " + + "FROM UserEntity u") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java index 436062ef4c..9a0add31ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -36,4 +38,7 @@ public interface UserSettingsRepository extends JpaRepository findByTypeAndPathExisting(@Param("type") String type, @Param("path") String[] path); + @Query("SELECT s FROM UserSettingsEntity s WHERE s.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index 6220d9b717..ed0c5155b9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -19,6 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.UserFields; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.page.PageData; @@ -30,6 +33,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeFilter; import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.common.data.widget.WidgetsBundleWidget; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.WidgetTypeDetailsEntity; import org.thingsboard.server.dao.model.sql.WidgetTypeInfoEntity; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; @@ -53,7 +57,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Component @SqlDao -public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao { +public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao, TenantEntityDao { @Autowired private WidgetTypeRepository widgetTypeRepository; @@ -255,10 +259,24 @@ public class JpaWidgetTypeDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(widgetTypeRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.WIDGET_TYPE; } + @Override + public ObjectType getType() { + return ObjectType.WIDGET_TYPE; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 290e949c38..5c9b4127f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -19,6 +19,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.page.PageData; @@ -26,6 +28,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.data.widget.WidgetsBundleFilter; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; @@ -44,7 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Component @SqlDao -public class JpaWidgetsBundleDao extends JpaAbstractDao implements WidgetsBundleDao { +public class JpaWidgetsBundleDao extends JpaAbstractDao implements WidgetsBundleDao, TenantEntityDao { @Autowired private WidgetsBundleRepository widgetsBundleRepository; @@ -155,7 +158,17 @@ public class JpaWidgetsBundleDao extends JpaAbstractDao findByImageLink(String imageUrl, int limit) { - return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + } + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(widgetsBundleRepository.findAllFields(DaoUtil.toPageable(pageLink))); } @Override @@ -163,4 +176,9 @@ public class JpaWidgetsBundleDao extends JpaAbstractDao findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields(w.id, w.createdTime, w.tenantId," + + "w.name, w.version) FROM WidgetTypeEntity w") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index 8de78986ae..97b2f52d63 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; @@ -139,4 +140,8 @@ public interface WidgetsBundleRepository extends JpaRepository findByImageUrl(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields(w.id, w.createdTime, w.tenantId," + + "w.alias, w.version) FROM WidgetsBundleEntity w") + Page findAllFields(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java index 89533106f3..3a024e1687 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java @@ -15,7 +15,11 @@ */ package org.thingsboard.server.dao.sql.widget; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetEntity; @@ -26,4 +30,7 @@ public interface WidgetsBundleWidgetRepository extends JpaRepository findAllByWidgetsBundleId(UUID widgetsBundleId); + @Query("SELECT w FROM WidgetsBundleWidgetEntity w WHERE w.widgetsBundleId IN (SELECT b.id FROM WidgetsBundleEntity b WHERE b.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 18364c6b1b..b134b6308d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -@SuppressWarnings("UnstableApiUsage") @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { @@ -119,4 +118,5 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index 0be078a89f..0e9cebd150 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -33,14 +33,18 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.timeseries.TsLatestCacheKey; import org.thingsboard.server.dao.util.SqlTsLatestAnyDaoCachedRedis; import java.util.List; +import java.util.Map; import java.util.Optional; @Slf4j @@ -167,4 +171,9 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public PageData findAllLatest(PageLink pageLink) { + return null; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 44859c26e7..fd1b180eef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -24,6 +24,7 @@ import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,8 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; @@ -185,6 +188,11 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); } + @Override + public PageData findAllLatest(PageLink pageLink) { + return DaoUtil.pageToPageData(tsKvLatestRepository.findAll(DaoUtil.toPageable(pageLink, "entityId", "key"))); + } + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); return Futures.transformAsync(future, entryList -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java index b01d1c4ea0..7194ae957c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -22,6 +22,9 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -92,4 +95,9 @@ public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); } + @Override + public PageData findAll(PageLink pageLink) { + return DaoUtil.pageToPageData(keyDictionaryRepository.findAll(DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java index 17e24ea5e5..0667f4315e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.dao.sqlts.dictionary; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -25,5 +28,7 @@ public interface KeyDictionaryRepository extends JpaRepository findByKeyId(int keyId); + @Query("SELECT e FROM KeyDictionaryEntry e ORDER BY e.keyId ASC") + Page findAll(Pageable pageable); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 38bfde0acf..833ffd185e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sqlts.latest; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -41,4 +43,7 @@ public interface TsKvLatestRepository extends JpaRepository findAllKeysByEntityIds(@Param("entityIds") List entityIds); + @Query("SELECT e FROM TsKvLatestEntity e ORDER BY e.entityId ASC, e.key ASC") + Page findAll(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java index 5fb5ff28c4..40ec208f08 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java @@ -37,10 +37,10 @@ public interface TenantDao extends Dao { * @return saved tenant object */ Tenant save(TenantId tenantId, Tenant tenant); - + /** * Find tenants by page link. - * + * * @param pageLink the page link * @return the list of tenant objects */ @@ -51,4 +51,5 @@ public interface TenantDao extends Dao { PageData findTenantsIds(PageLink pageLink); List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 756b73d88b..cb731a782f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -26,6 +26,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; @@ -38,6 +40,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.Validator; @@ -54,7 +57,6 @@ import static org.thingsboard.server.common.data.StringUtils.isBlank; /** * @author Andrew Shvayka */ -@SuppressWarnings("UnstableApiUsage") @Service @Slf4j public class BaseTimeseriesService implements TimeseriesService { @@ -89,6 +91,9 @@ public class BaseTimeseriesService implements TimeseriesService { @Autowired private EntityViewService entityViewService; + @Autowired + private EdqsService edqsService; + @Override public ListenableFuture> findAllByQueries(TenantId tenantId, EntityId entityId, List queries) { validate(entityId); @@ -190,14 +195,21 @@ public class BaseTimeseriesService implements TimeseriesService { public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { List> futures = new ArrayList<>(tsKvEntries.size()); for (TsKvEntry tsKvEntry : tsKvEntries) { - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + futures.add(doSaveLatest(tenantId, entityId, tsKvEntry)); } return Futures.allAsList(futures); } private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); + futures.add(Futures.transform(doSaveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); + } + + private ListenableFuture doSaveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + return Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), version -> { + edqsService.onUpdate(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); + return version; + }, MoreExecutors.directExecutor()); } private void saveWithoutLatestAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { @@ -248,7 +260,7 @@ public class BaseTimeseriesService implements TimeseriesService { List> futures = new ArrayList<>(keys.size()); for (String key : keys) { DeleteTsKvQuery query = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false); - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } return Futures.allAsList(futures); } @@ -269,10 +281,20 @@ public class BaseTimeseriesService implements TimeseriesService { private void deleteAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, DeleteTsKvQuery query) { futures.add(Futures.transform(timeseriesDao.remove(tenantId, entityId, query), v -> null, MoreExecutors.directExecutor())); if (query.getDeleteLatest()) { - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } } + private ListenableFuture doRemove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return Futures.transform(timeseriesLatestDao.removeLatest(tenantId, entityId, query), result -> { + if (result.isRemoved()) { + Long version = result.getVersion(); + edqsService.onDelete(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, query.getKey(), version)); + } + return result; + }, MoreExecutors.directExecutor()); + } + private static void validate(EntityId entityId) { Validator.validateEntityId(entityId, id -> "Incorrect entityId " + id); } @@ -302,4 +324,5 @@ public class BaseTimeseriesService implements TimeseriesService { throw new IncorrectParameterException("Incorrect DeleteTsKvQuery. Key can't be empty"); } } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 01c91d4801..99bf57a1db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -36,13 +36,17 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -99,6 +103,11 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } + @Override + public PageData findAllLatest(PageLink pageLink) { + return null; + } + @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 9f62fd033a..1372f2f5d8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -22,8 +22,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import java.util.List; +import java.util.Map; import java.util.Optional; public interface TimeseriesLatestDao { @@ -49,4 +53,6 @@ public interface TimeseriesLatestDao { List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + + PageData findAllLatest(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java index ce71b733e5..d0da2a3943 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java @@ -19,10 +19,11 @@ import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface ApiUsageStateDao extends Dao { +public interface ApiUsageStateDao extends Dao, TenantEntityDao { /** * Save or update usage record object @@ -50,4 +51,5 @@ public interface ApiUsageStateDao extends Dao { void deleteApiUsageStateByTenantId(TenantId tenantId); void deleteApiUsageStateByEntityId(EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index d901ae950e..9512120a3b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -113,6 +113,14 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A ApiUsageState saved = apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + eventPublisher.publishEvent(SaveEntityEvent.builder() + .tenantId(saved.getTenantId()) + .entityId(saved.getId()) + .entity(saved) + .created(true) + .broadcastEvent(false) + .build()); + List apiUsageStates = new ArrayList<>(); apiUsageStates.add(new BasicTsKvEntry(saved.getCreatedTime(), new StringDataEntry(ApiFeature.TRANSPORT.getApiStateKey(), ApiUsageStateValue.ENABLED.name()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index 42add1bbe2..78b9455586 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 7d8018cdd8..f8a0b43741 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -77,7 +77,6 @@ import java.util.UUID; import static org.junit.Assert.assertNotNull; - @RunWith(SpringRunner.class) @ContextConfiguration(classes = AbstractServiceTest.class, loader = AnnotationConfigContextLoader.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @@ -131,7 +130,7 @@ public abstract class AbstractServiceTest { } public JsonNode readFromResource(String resourceName) throws IOException { - try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)){ + try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)) { return JacksonUtil.fromBytes(Objects.requireNonNull(is).readAllBytes()); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java index 30110a2741..e06d83682a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java @@ -21,16 +21,31 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; +import static org.thingsboard.server.common.data.ObjectType.EVENT; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.OAUTH2_CLIENT; +import static org.thingsboard.server.common.data.ObjectType.OAUTH2_DOMAIN; +import static org.thingsboard.server.common.data.ObjectType.OAUTH2_MOBILE; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.TENANT; +import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE; @Slf4j @DaoSqlTest @@ -88,4 +103,20 @@ public class EntityDaoRegistryTest extends AbstractServiceTest { } } + @Test + public void givenAllTenantEntityDaos_whenFindAllByTenantId_thenOk() { + Set ignored = EnumSet.of(TENANT, TENANT_PROFILE, RELATION, EVENT, ATTRIBUTE_KV, LATEST_TS_KV, AUDIT_LOG, + OAUTH2_CLIENT, OAUTH2_DOMAIN, OAUTH2_MOBILE); + for (ObjectType type : ObjectType.values()) { + if (ignored.contains(type)) { + continue; + } + + TenantEntityDao dao = assertDoesNotThrow(() -> entityDaoRegistry.getTenantEntityDao(type)); + assertDoesNotThrow(() -> { + dao.findAllByTenantId(tenantId, new PageLink(100)); + }); + } + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java index 2a0210f8c8..b880871bd6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -29,6 +29,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit4.SpringRunner; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.QueryContext; import java.util.List; import java.util.UUID; @@ -48,7 +50,7 @@ import static org.mockito.Mockito.times; public class DefaultQueryLogComponentTest { private TenantId tenantId; - private QueryContext ctx; + private SqlQueryContext ctx; @SpyBean private DefaultQueryLogComponent queryLog; @@ -56,7 +58,7 @@ public class DefaultQueryLogComponentTest { @Before public void setUp() { tenantId = new TenantId(UUID.fromString("97275c1c-9cf2-4d25-a68d-933031158f84")); - ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); } @Test diff --git a/edqs/pom.xml b/edqs/pom.xml new file mode 100644 index 0000000000..bf83c75b32 --- /dev/null +++ b/edqs/pom.xml @@ -0,0 +1,260 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0PE-SNAPSHOT + thingsboard + + edqs + jar + + ThingsBoard Entity Data Query Service Application + https://thingsboard.io + ThingsBoard Professional Edition: IoT Platform - Device management, data collection, processing and visualization + + + + UTF-8 + ${basedir}/.. + java + false + process-resources + package + edqs + ${project.build.directory}/windows + true + ThingsBoard Entity Data Query Service + org.thingsboard.server.edqs.ThingsboardEdqsApplication + + + + + org.thingsboard.common + edqs + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.apache.curator + curator-recipes + + + com.google.protobuf + protobuf-java + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + + com.sun.winsw + winsw + bin + exe + provided + + + org.thingsboard + tools + test + + + org.springframework.security + spring-security-test + test + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.dbunit + dbunit + test + + + com.github.springtestdbunit + spring-test-dbunit + test + + + org.testcontainers + cassandra + test + + + org.testcontainers + postgresql + test + + + org.testcontainers + jdbc + test + + + org.java-websocket + Java-WebSocket + test + + + org.eclipse.milo + sdk-server + test + + + org.assertj + assertj-core + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + true + + edqs.yml + + + + ${project.basedir}/src/main/resources + false + + edqs.yml + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + thingsboard + + + **/nosql/*Test.java + + + **/*Test.java + **/*TestSuite.java + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-winsw-service + package + + + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + org.codehaus.mojo + build-helper-maven-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java new file mode 100644 index 0000000000..ea8af83310 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class DummyQueueRoutingInfoService implements QueueRoutingInfoService { + + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java new file mode 100644 index 0000000000..5037652e5c --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class DummyTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + return null; + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java new file mode 100644 index 0000000000..1157f74de1 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.thingsboard.server.edqs.repo.EdqRepository; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server.edqs", "org.thingsboard.server.queue.edqs", "org.thingsboard.server.queue.discovery", "org.thingsboard.server.queue.kafka", + "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment"}) +@Slf4j +public class ThingsboardEdqsApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "edqs"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardEdqsApplication.class, updateArguments(args)); + } + + // @Bean + public ApplicationRunner runner(CSVLoader loader, EdqRepository edqRepository) { + return args -> { +// long startTs = System.currentTimeMillis(); +// var loader = new TenantRepoLoader(new TenantRepo(TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")))); +// loader.load(); +// log.info("Loaded all in {} ms", System.currentTimeMillis() - startTs); + + + +// log.info("Compressed {} strings/json, Before: {}, After: {}", +// CompressedStringDataPoint.cnt.get(), +// CompressedStringDataPoint.uncompressedLength.get(), +// CompressedStringDataPoint.compressedLength.get()); +// +// log.info("Deduplicated {} short and {} long strings", +// TbStringPool.size(), TbBytePool.size()); +// +// var tenantId = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); +// var customerId = new CustomerId(UUID.fromString("fcbf2f50-d0d9-11ea-bea3-177755191a6e")); +// System.gc(); +// +// while (true) { +// EntityTypeFilter filter = new EntityTypeFilter(); +// filter.setEntityType(EntityType.DEVICE); +// var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); +// +// var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); +// var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); +// KeyFilter nameFilter = new KeyFilter(); +// nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); +// var predicate = new StringFilterPredicate(); +// predicate.setIgnoreCase(false); +// predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); +// predicate.setValue(new FilterPredicateValue<>("LoRa-")); +// nameFilter.setPredicate(predicate); +// nameFilter.setValueType(EntityKeyValueType.STRING); +// +// EntityDataQuery edq = new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); +// var result = edqRepository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, edq, false); +// log.info("Device count: {}", result.getTotalElements()); +// log.info("First: {}", result.getData().get(0).getEntityId()); +// log.info("Last: {}", result.getData().get(19).getEntityId()); +// +// pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.ASC)); +// result = edqRepository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, edq, false); +// log.info("Device count: {}", result.getTotalElements()); +// log.info("First: {}", result.getData().get(0).getEntityId()); +// log.info("Last: {}", result.getData().get(19).getEntityId()); +// +// result.getData().forEach(data -> { +// System.err.println(data.getEntityId() + ":"); +// data.getLatest().forEach((type, values) -> { +// System.err.println(type); +// values.forEach((key, tsValue) -> { +// System.err.println(key + " = " + tsValue.getValue()); +// }); +// }); +// System.err.println(); +// }); +// Thread.sleep(5000); +// } + }; + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } + +} diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml new file mode 100644 index 0000000000..f5c83178c3 --- /dev/null +++ b/edqs/src/main/resources/edqs.yml @@ -0,0 +1,364 @@ +# +# Copyright © 2016-2024 ThingsBoard, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Application info parameters +app: + # Application version + version: "@project.version@" + +# Zookeeper connection parameters +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. + # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. + # The delay is recommended because the initialization of rule chain actors is time-consuming. Avoiding unnecessary recalculations during a restart can enhance system performance and stability. + recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" + +# SQL DAO Configuration parameters +spring: + main: + web-application-type: "none" + +# Queue configuration parameters +queue: + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) + prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). + in_memory: + stats: + # For debug level + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + edqs: + enabled: "${TB_EDQS_ENABLED:true}" + mode: "${TB_EDQS_MODE:local}" + partitions: "${TB_EDQS_PARTITIONS:12}" + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data + kafka: + # Kafka Bootstrap nodes in "host:port" format + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + ssl: + # Enable/Disable SSL Kafka communication + enabled: "${TB_KAFKA_SSL_ENABLED:false}" + # The location of the trust store file + truststore.location: "${TB_KAFKA_SSL_TRUSTSTORE_LOCATION:}" + # The password of trust store file if specified + truststore.password: "${TB_KAFKA_SSL_TRUSTSTORE_PASSWORD:}" + # The location of the key store file. This is optional for the client and can be used for two-way authentication for the client + keystore.location: "${TB_KAFKA_SSL_KEYSTORE_LOCATION:}" + # The store password for the key store file. This is optional for the client and only needed if ‘ssl.keystore.location’ is configured. Key store password is not supported for PEM format + keystore.password: "${TB_KAFKA_SSL_KEYSTORE_PASSWORD:}" + # The password of the private key in the key store file or the PEM key specified in ‘keystore.key’ + key.password: "${TB_KAFKA_SSL_KEY_PASSWORD:}" + # The number of acknowledgments the producer requires the leader to have received before considering a request complete. This controls the durability of records that are sent. The following settings are allowed:0, 1 and all + acks: "${TB_KAFKA_ACKS:all}" + # Number of retries. Resend any record whose send fails with a potentially transient error + retries: "${TB_KAFKA_RETRIES:1}" + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # Default batch size. This setting gives the upper bound of the batch size to be sent + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + # The maximum size of a request in bytes. This setting will limit the number of record batches the producer will send in a single request to avoid sending huge requests + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + # The maximum number of unacknowledged requests the client will send on a single connection before blocking + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + # The total bytes of memory the producer can use to buffer records waiting to be sent to the server + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + # The multiple copies of data over the multiple brokers of Kafka + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + # The maximum delay between invocations of poll() method when using consumer group management. This places an upper bound on the amount of time that the consumer can be idle before fetching more records + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + # The maximum number of records returned in a single call of poll() method + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + # The maximum amount of data per-partition the server will return. Records are fetched in batches by the consumer + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + # The maximum amount of data the server will return. Records are fetched in batches by the consumer + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none + # Enable/Disable using of Confluent Cloud + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + # The endpoint identification algorithm used by clients to validate server hostname. The default value is https + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + # The mechanism used to authenticate Schema Registry requests. SASL/PLAIN should only be used with TLS/SSL as a transport layer to ensure that clear passwords are not transmitted on the wire without encryption + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + # Using JAAS Configuration for specifying multiple SASL mechanisms on a broker + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + tb_ota_package: + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + - key: max.poll.records + # Example of specific consumer properties value per topic + value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + # Example of specific consumer properties value per topic for VC + - key: max.poll.interval.ms + # Example of specific consumer properties value per topic for VC + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_housekeeper: + # Consumer properties for Housekeeper tasks topic + - key: max.poll.records + # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_MAX_POLL_RECORDS:1}" + tb_housekeeper.reprocessing: + # Consumer properties for Housekeeper reprocessing topic + - key: max.poll.records + # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + # Kafka properties for Rule Engine + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Core topics + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Transport Api topics + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Notifications topics + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for JS Executor topics + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + # Kafka properties for OTA updates topic + ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Version Control topic + version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Integration Api topics + integration-api: "${TB_QUEUE_KAFKA_INTEGRATION_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time + housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + consumer-stats: + # Prints lag between consumer group offset and last messages offset in Kafka topics + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + # Statistics printing interval for Kafka's consumer-groups stats + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + # Time to wait for the stats-loading requests to Kafka to finish + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + transport_api: + # Topic used to consume api requests from transport microservices + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + # Topic used to produce api responses to transport microservices + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + # Maximum pending api requests from transport microservices to be handled by server + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout in milliseconds to handle api request from transport microservice by server + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + # Amount of threads used to invoke callbacks + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + # Amount of threads used for transport API requests + max_core_handler_threads: "${TB_QUEUE_TRANSPORT_MAX_CORE_HANDLER_THREADS:16}" + # Interval in milliseconds to poll api requests from transport microservices + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + # Interval in milliseconds to poll api response from transport microservices + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + # Default topic name of Kafka, RabbitMQ, etc. queue + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # Interval in milliseconds to poll messages by Core microservices + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + # Amount of partitions used by Core microservices + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + # Timeout for processing a message pack by Core microservices + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + # Enable/disable a separate consumer per partition for Core queue + consumer-per-partition: "${TB_QUEUE_CORE_CONSUMER_PER_PARTITION:true}" + ota: + # Default topic name for OTA updates + topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + # The interval of processing the OTA updates for devices. Used to avoid any harm to the network due to many parallel OTA updates + pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + # The size of OTA updates notifications fetched from the queue. The queue stores pairs of firmware and device ids + pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + # Stats topic name for queue Kafka, RabbitMQ, etc. + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + # Enable/disable statistics for Core microservices + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + # Statistics printing interval for Core microservices + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" + # Topic name for Housekeeper tasks to be reprocessed + reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" + # Poll interval for topics related to Housekeeper + poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" + # Timeout in milliseconds for task processing. Tasks that fail to finish on time will be submitted for reprocessing + task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" + # Comma-separated list of task types that shouldn't be processed. Available task types: + # DELETE_ATTRIBUTES, DELETE_TELEMETRY (both DELETE_LATEST_TS and DELETE_TS_HISTORY will be disabled), + # DELETE_LATEST_TS, DELETE_TS_HISTORY, DELETE_EVENTS, DELETE_ALARMS, UNASSIGN_ALARMS + disabled-task-types: "${TB_HOUSEKEEPER_DISABLED_TASK_TYPES:}" + # Delay in milliseconds between tasks reprocessing + task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:3000}" + # Maximum amount of task reprocessing attempts. After exceeding, the task will be dropped + max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}" + stats: + # Enable/disable statistics for Housekeeper + enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" + # Statistics printing interval for Housekeeper + print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" + + vc: + # Default topic name for Kafka, RabbitMQ, etc. + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + # Number of partitions to associate with this queue. Used for scaling the number of messages that can be processed in parallel + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + # Interval in milliseconds between polling of the messages if no new messages arrive + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + # Timeout before retrying all failed and timed-out messages from the processing pack + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" + # Timeout for a request to VC-executor (for a request for the version of the entity, for a commit charge, etc.) + request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:180000}" + # Queue settings for Kafka, RabbitMQ, etc. Limit for single message size + msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}" + # JS max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS execution max request timeout + max_exec_requests_timeout: "${REMOTE_JS_MAX_EXEC_REQUEST_TIMEOUT:2000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + rule-engine: + # Deprecated. It will be removed in the nearest releases + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # Interval in milliseconds to poll messages by Rule Engine + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + # Timeout for processing a message pack of Rule Engine + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}" + stats: + # Enable/disable statistics for Rule Engine + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + # Statistics printing interval for Rule Engine + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" + # Max length of the error message that is printed by statistics + max-error-message-length: "${TB_QUEUE_RULE_ENGINE_MAX_ERROR_MESSAGE_LENGTH:4096}" + # After a queue is deleted (or the profile's isolation option was disabled), Rule Engine will continue reading related topics during this period before deleting the actual topics + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" + # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion + management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" + transport: + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + # Interval in milliseconds to poll messages + poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + integration: + # Name of hash function used for consistent hash ring in Cluster Mode. See architecture docs for more details. Valid values - murmur3_32, murmur3_128 or sha256 + partitions: "${TB_QUEUE_INTEGRATION_PARTITIONS:3}" + # Default notification topic name used by queue + notifications_topic: "${TB_QUEUE_INTEGRATION_NOTIFICATIONS_TOPIC:tb_ie.notifications}" + # Default downlink topic name used by queue + downlink_topic: "${TB_QUEUE_INTEGRATION_DOWNLINK_TOPIC:tb_ie.downlink}" + # Default uplink topic name used by queue + uplink_topic: "${TB_QUEUE_INTEGRATION_UPLINK_TOPIC:tb_ie.uplink}" + # Interval in milliseconds to poll messages by integrations + poll_interval: "${TB_QUEUE_INTEGRATION_POLL_INTERVAL_MS:25}" + # Timeout for processing a message pack by integrations + pack-processing-timeout: "${TB_QUEUE_INTEGRATION_PACK_PROCESSING_TIMEOUT_MS:10000}" + integration_api: + # Default Integration Api request topic name used by queue + requests_topic: "${TB_QUEUE_INTEGRATION_EXECUTOR_API_REQUEST_TOPIC:tb_ie.api.requests}" + # Default Integration Api response topic name used by queue + responses_topic: "${TB_QUEUE_INTEGRATION_EXECUTOR_API_RESPONSE_TOPIC:tb_ie.api.responses}" + # Maximum pending api requests from integration executor to be handled by server< + max_pending_requests: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout in milliseconds to handle api request from integration executor microservice by server + max_requests_timeout: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_REQUEST_TIMEOUT:10000}" + # Amount of threads used to invoke callbacks + max_callback_threads: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_CALLBACK_THREADS:10}" + # Interval in milliseconds to poll api requests from integration executor microservices + request_poll_interval: "${TB_QUEUE_INTEGRATION_EXECUTOR_REQUEST_POLL_INTERVAL_MS:25}" + # Interval in milliseconds to poll api response from integration executor microservices + response_poll_interval: "${TB_QUEUE_INTEGRATION_EXECUTOR_RESPONSE_POLL_INTERVAL_MS:25}" + +# General service parameters +service: + type: "${TB_SERVICE_TYPE:edqs}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + edqs: + label: "${TB_EDQS_LABEL:}" # services with the same label will share the list of partitions + +# Metrics parameters +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + system_info: + # Persist frequency of system info (CPU, memory usage, etc.) in seconds + persist_frequency: "${METRICS_SYSTEM_INFO_PERSIST_FREQUENCY_SECONDS:60}" + # TTL in days for system info timeseries + ttl: "${METRICS_SYSTEM_INFO_TTL_DAYS:7}" + +# General management parameters +management: + endpoints: + web: + exposure: + # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). + include: '${METRICS_ENDPOINTS_EXPOSE:info}' + health: + elasticsearch: + # Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck + enabled: "false" diff --git a/edqs/src/main/resources/logback.xml b/edqs/src/main/resources/logback.xml new file mode 100644 index 0000000000..09aa2b1ecb --- /dev/null +++ b/edqs/src/main/resources/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java new file mode 100644 index 0000000000..f6f3fcc60e --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -0,0 +1,296 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.SchedulerEventId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.scheduler.SchedulerEvent; +import org.thingsboard.server.edqs.processor.EdqsConverter; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@RunWith(SpringRunner.class) +@Configuration +@ComponentScan({"org.thingsboard.server.edqs.repo"}) +@EntityScan("org.thingsboard.server.edqs") +@TestPropertySource(locations = {"classpath:edq-test.properties"}) +@TestExecutionListeners({ + DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class}) +public abstract class AbstractEDQTest { + + @Autowired + protected InMemoryEdqRepository repository; + + protected final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + protected final CustomerId customerId = new CustomerId(UUID.randomUUID()); + + protected final UUID defaultAssetProfileId = UUID.randomUUID(); + protected final UUID defaultDeviceProfileId = UUID.randomUUID(); + + @Before + public final void before() { + AssetProfile ap = new AssetProfile(new AssetProfileId(defaultAssetProfileId)); + ap.setName("default"); + ap.setDefault(true); + addOrUpdate(EntityType.ASSET_PROFILE, ap); + + DeviceProfile dp = new DeviceProfile(new DeviceProfileId(defaultDeviceProfileId)); + dp.setName("default"); + dp.setDefault(true); + dp.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, dp); + + createCustomer(customerId.getId(), null, "Customer A"); + } + + @After + public final void after() { + repository.clear(); + } + + protected void createCustomer(UUID id, UUID parentCustomerId, String title) { + Customer entity = new Customer(); + entity.setId(new CustomerId(id)); + entity.setTitle(title); + entity.setOwnerId(parentCustomerId != null ? new CustomerId(parentCustomerId) : tenantId); + addOrUpdate(EntityType.CUSTOMER, entity); + } + + protected UUID createGroup(EntityType entityType, String groupName) { + return createGroup(null, entityType, groupName); + } + + protected UUID createGroup(UUID customerId, EntityType entityType, String groupName) { + EntityGroup eg = new EntityGroup(); + eg.setId(new EntityGroupId(UUID.randomUUID())); + eg.setType(entityType); + eg.setName(groupName); + eg.setOwnerId(customerId != null ? new CustomerId(customerId) : tenantId); + addOrUpdate(EntityType.ENTITY_GROUP, eg); + return eg.getId().getId(); + } + + protected UUID createDevice(String name) { + return createDevice(null, defaultDeviceProfileId, name); + } + + protected UUID createDevice(CustomerId customerId, String name) { + return createDevice(customerId.getId(), defaultDeviceProfileId, name); + } + + protected UUID createDevice(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Device entity = new Device(); + entity.setId(new DeviceId(entityId)); + if (profileId != null) { + entity.setDeviceProfileId(new DeviceProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createDashboard(String name) { + return createDashboard(null, name); + } + + protected UUID createDashboard(UUID customerId, String name) { + UUID entityId = UUID.randomUUID(); + Dashboard entity = new Dashboard(); + entity.setId(new DashboardId(entityId)); + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setTitle(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createView(String name) { + return createView(null, "default", name); + } + + protected UUID createView(CustomerId customerId, String name) { + return createView(customerId.getId(), "default", name); + } + + protected UUID createView(UUID customerId, String type, String name) { + UUID entityId = UUID.randomUUID(); + EntityView entity = new EntityView(); + entity.setId(new EntityViewId(entityId)); + entity.setType(type); + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ENTITY_VIEW, entity); + return entityId; + } + + protected UUID createEdge(String name) { + return createEdge(null, "default", name); + } + + protected UUID createEdge(CustomerId customerId, String name) { + return createEdge(customerId.getId(), "default", name); + } + + protected UUID createEdge(UUID customerId, String type, String name) { + UUID id = UUID.randomUUID(); + Edge edge = new Edge(); + edge.setId(new EdgeId(id)); + edge.setTenantId(tenantId); + if (customerId != null) { + edge.setCustomerId(new CustomerId(customerId)); + } + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + addOrUpdate(EntityType.EDGE, edge); + return id; + } + + protected UUID createSchedulerEvent(String type, EntityId originatorId, String name) { + return createSchedulerEvent(null, type, originatorId, name); + } + + protected UUID createSchedulerEvent(UUID customerId, String type, EntityId originatorId, String name) { + UUID id = UUID.randomUUID(); + SchedulerEvent schedulerEvent = new SchedulerEvent(); + schedulerEvent.setId(new SchedulerEventId(id)); + schedulerEvent.setTenantId(tenantId); + if (customerId != null) { + schedulerEvent.setCustomerId(new CustomerId(customerId)); + } + schedulerEvent.setType(type); + schedulerEvent.setName(name); + schedulerEvent.setConfiguration(JacksonUtil.newObjectNode()); + schedulerEvent.setSchedule(JacksonUtil.newObjectNode()); + schedulerEvent.setOriginatorId(originatorId); + schedulerEvent.setCreatedTime(42L); + addOrUpdate(EntityType.SCHEDULER_EVENT, schedulerEvent); + return id; + } + + protected UUID createAsset(String name) { + return createAsset(null, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, String name) { + return createAsset(customerId, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Asset entity = new Asset(); + entity.setId(new AssetId(entityId)); + if (profileId != null) { + entity.setAssetProfileId(new AssetProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ASSET, entity); + return entityId; + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, String type) { + createRelation(fromType, fromId, toType, toId, RelationTypeGroup.COMMON, type); + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, RelationTypeGroup group, String type) { + repository.get(tenantId).addOrUpdate(new EntityRelation(EntityIdFactory.getByTypeAndUuid(fromType, fromId), EntityIdFactory.getByTypeAndUuid(toType, toId), type, group)); + } + + + protected boolean checkContains(PageData data, UUID entityId) { + return data.getData().stream().anyMatch(r -> r.getEntityId().getId().equals(entityId)); + } + + protected List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value) { + KeyFilter filter = new KeyFilter(); + filter.setKey(new EntityKey(keyType, key)); + filter.setValueType(EntityKeyValueType.STRING); + StringFilterPredicate predicate = new StringFilterPredicate(); + predicate.setValue(FilterPredicateValue.fromString(value)); + predicate.setOperation(operation); + predicate.setIgnoreCase(true); + filter.setPredicate(predicate); + return Collections.singletonList(filter); + } + + protected void addOrUpdate(EntityType entityType, Object entity) { + addOrUpdate(EdqsConverter.toEntity(entityType, entity)); + } + + protected void addOrUpdate(EdqsObject edqsObject) { + repository.get(tenantId).addOrUpdate(edqsObject); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java new file mode 100644 index 0000000000..de409ca293 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.UUID; + +public class ApiUsageStateFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + Tenant entity = new Tenant(); + entity.setId(tenantId); + entity.setTitle("test tenant"); + addOrUpdate(EntityType.TENANT, entity); + } + + @After + public void tearDown() { + } + + @Test + public void testFindCustomerApiUsageState() { + UUID customerId = UUID.randomUUID(); + createCustomer(customerId, null, "Customer A"); + + ApiUsageState apiUsageState = buildApiUsageState(customerId); + addOrUpdate(EntityType.API_USAGE_STATE, apiUsageState); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new CustomerId(customerId)), false); + + Assert.assertEquals(1, result.getTotalElements()); + var customer = result.getData().get(0); + Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + private ApiUsageState buildApiUsageState(UUID customerId) { + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setId(new ApiUsageStateId(UUID.randomUUID())); + apiUsageState.setTenantId(tenantId); + apiUsageState.setEntityId(new CustomerId(customerId)); + apiUsageState.setTransportState(ApiUsageStateValue.ENABLED); + apiUsageState.setReExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setJsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setTbelExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setDbStorageState(ApiUsageStateValue.ENABLED); + apiUsageState.setSmsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setEmailExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setAlarmExecState(ApiUsageStateValue.ENABLED); + return apiUsageState; + } + + private static EntityDataQuery getEntityDataQuery(CustomerId customerId) { + ApiUsageStateFilter filter = new ApiUsageStateFilter(); + filter.setCustomerId(customerId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("Customer A")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java new file mode 100644 index 0000000000..2983ce57c3 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java @@ -0,0 +1,223 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class AssetSearchQueryFilterTest extends AbstractEDQTest { + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(null, assetProfileId.getId(), "root"); + UUID asset1 = createAsset(null, assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(null, assetProfileId.getId(), "A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets with max level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + + // find all assets with asset type = default + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all assets last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets last level only, level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindTenantAssetsWithGroupPermissionsOnly() { + UUID eg1 = createGroup(EntityType.ASSET, "Group A"); + + UUID root = createAsset("root"); + UUID asset1 = createAsset("A1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ASSET, asset1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID asset2 = createAsset("A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ASSET, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindCustomerAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(customerId.getId(), assetProfileId.getId(), "root"); + UUID asset1 = createAsset(customerId.getId(), assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(customerId.getId(), assetProfileId.getId(), "A2"); + UUID asset3 = createAsset(customerId.getId(), defaultAssetProfileId, "A3"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset3, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset with profile "Office" + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office", "default")); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + Assert.assertTrue(checkContains(relationsResult, asset3)); + + // find all assets with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerAssetsWithGroupPermission() { + UUID eg1 = createGroup(EntityType.ASSET, "Group A"); + + UUID root = createAsset(customerId.getId(), defaultAssetProfileId, "root"); + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId,"A1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ASSET, asset1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId,"A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ASSET, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindCustomerAssetsWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + UUID asset1 = createAsset(subCustomer.getId(), defaultAssetProfileId,"A1"); + UUID eg1 = createGroup(subCustomer.getId(), EntityType.ASSET, "Group A"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ASSET, asset1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + UUID root = createAsset(customerId.getId(), defaultAssetProfileId, "root"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId,"A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset2, "Contains"); + + // find all assets group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ASSET, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + } + + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List assetTypes) { + AssetSearchQueryFilter filter = new AssetSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setAssetTypes(assetTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "A"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java new file mode 100644 index 0000000000..6048669d66 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AssetTypeFilterTest extends AbstractEDQTest { + + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + private final AssetProfileId assetProfileId2 = new AssetProfileId(UUID.randomUUID()); + private Asset asset; + private Asset asset2; + private Asset asset3; + + @Before + public void setUp() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + AssetProfile assetProfile2 = new AssetProfile(assetProfileId2); + assetProfile2.setName("Street"); + assetProfile2.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile2); + + asset = buildAsset(assetProfileId, "Office 1"); + asset2 = buildAsset(assetProfileId, "Office 2"); + asset3 = buildAsset(assetProfileId2, "Abbey Road"); + + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(EntityType.ASSET, asset2); + addOrUpdate(EntityType.ASSET, asset3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantAsset() { + // find asset with type "Office" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(Collections.singletonList("Office"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Office 1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(asset.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Office" and "Street" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office", "Street"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var third = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Abbey Road")).findAny(); + assertThat(third).isPresent(); + assertThat(third.get().getEntityId()).isEqualTo(asset3.getId()); + assertThat(third.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Supermarket" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with name "%Office%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), "%Office%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with name "Office 1" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), "Office 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find asset with name "%Super%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), "%Super%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with key filter: name contains "Office" + KeyFilter containsNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "office", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with key filter: name starts with "office" and matches case + KeyFilter startsWithNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "office", false); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerAsset() { + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(new LatestTsKv(asset.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + asset.setCustomerId(customerId); + addOrUpdate(EntityType.ASSET, asset); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(asset.getId(), first.getEntityId()); + Assert.assertEquals("Office 1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Asset buildAsset(AssetProfileId assetProfileId, String assetName) { + Asset asset = new Asset(); + asset.setId(new AssetId(UUID.randomUUID())); + asset.setTenantId(tenantId); + asset.setAssetProfileId(assetProfileId); + asset.setName(assetName); + asset.setCreatedTime(42L); + return asset; + } + + private static EntityDataQuery getAssetTypeQuery(List assetTypes, String assetNameRegex, List keyFilters) { + AssetTypeFilter filter = new AssetTypeFilter(); + filter.setAssetTypes(assetTypes); + filter.setAssetNameFilter(assetNameRegex); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getAssetNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java new file mode 100644 index 0000000000..bea0f57246 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java @@ -0,0 +1,241 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class DeviceSearchQueryFilterTest extends AbstractEDQTest { + private final DeviceProfileId deviceProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset( "A1"); + UUID asset2 = createAsset( "A2"); + UUID device1 = createDevice(null, deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(null, deviceProfileId.getId(), "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of asset A1 + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all devices with max level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices with asset type = default + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all devices last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device2)); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices last level only, level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindTenantDevicesWithGroupPermissionsOnly() { + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + + UUID asset1 = createAsset( "A1"); + UUID asset2 = createAsset( "A2"); + UUID device1 = createDevice(null, defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(null, defaultDeviceProfileId, "D2"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, device1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindCustomerDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of type "thermostat" + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat", "default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all assets with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerAssetsWithGroupPermission() { + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), deviceProfileId.getId(), "D1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, device1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindCustomerEdgesWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + UUID eg1 = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group A"); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(subCustomer.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, device1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID device2 = createDevice(subCustomer.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices with generic and group permission + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, subCustomer, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + } + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List deviceTypes) { + DeviceSearchQueryFilter filter = new DeviceSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setDeviceTypes(deviceTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "D"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java new file mode 100644 index 0000000000..e4c3a0ffad --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java @@ -0,0 +1,142 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.UUID; + +public class DeviceTypeFilterTest extends AbstractEDQTest { + + private final DeviceProfileId loraProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + DeviceProfile deviceProfile = new DeviceProfile(loraProfileId); + deviceProfile.setName("LoRa"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setDeviceProfileId(loraProfileId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("default"), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(loraProfileId); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceTypeQuery(String deviceType) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(Collections.singletonList(deviceType)); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java new file mode 100644 index 0000000000..15a0b8b4e4 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java @@ -0,0 +1,167 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; + +public class EdgeSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindDevicesManagesByTenant() { + UUID edge1 = createEdge("E1"); + UUID edge2 = createEdge("E2"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID device3 = createDevice("D3"); + + createRelation(EntityType.EDGE, edge1, EntityType.DEVICE, device1, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device2, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device3, "Manages"); + + // find devices managed by edge + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + + // find devices managed by edge with non-existing type + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 1, false, Arrays.asList("non-existing type")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, true, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + } + + @Test + public void testFindTenantEdgesWithGroupPermissionOnly() { + UUID eg1 = createGroup(EntityType.EDGE, "Group A"); + + UUID edge1 = createEdge("E1"); + UUID edge2 = createEdge("E2"); + createRelation(EntityType.TENANT, tenantId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.TENANT, tenantId.getId(), EntityType.EDGE, edge2, "Manages"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, edge1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.EDGE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, tenantId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + } + + @Test + public void testFindCustomerEdges() { + UUID edge1 = createEdge(customerId, "E1"); + UUID edge2 = createEdge(customerId, "E2"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge2, "Manages"); + + // find all edges managed by customer + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + Assert.assertTrue(checkContains(relationsResult, edge2)); + + // find all edges managed by customer with non-existing type + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("non existing")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerEdgesWithGroupPermission() { + UUID eg1 = createGroup(EntityType.EDGE, "Group A"); + + UUID edge1 = createEdge(customerId, "E1"); + UUID edge2 = createEdge(customerId, "E2"); + UUID edge3 = createEdge(customerId, "E3"); + + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, edge1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, edge2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge2, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge3, "Manages"); + + // find all entity views with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.EDGE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + Assert.assertTrue(checkContains(relationsResult, edge2)); + } + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List edgeTypes) { + EdgeSearchQueryFilter filter = new EdgeSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEdgeTypes(edgeTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "E"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java new file mode 100644 index 0000000000..8c48e0f1ea --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EdgeTypeFilterTest extends AbstractEDQTest { + + private Edge edge; + private Edge edge2; + private Edge edge3; + + + @Before + public void setUp() { + edge = buildEdge("default", "Edge 1"); + edge2 = buildEdge("default", "Edge 2"); + edge3 = buildEdge("edge v2", "Edge 3"); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEdges() { + // find edges with type "default" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(Collections.singletonList("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find edges with types "default" and "edge v2" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(Arrays.asList("default", "edge v2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 3")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(edge3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Edge%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), "%Edge%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Edge 1" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), "Edge 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Edge 4%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), "%Edge 4%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Edge" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Edge", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, List.of(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "edge" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "edge", false); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, List.of(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEdges() { + addOrUpdate(new LatestTsKv(edge.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + edge.setCustomerId(customerId); + edge2.setCustomerId(customerId); + edge3.setCustomerId(customerId); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Edge buildEdge(String type, String name) { + Edge edge = new Edge(); + edge.setId(new EdgeId(UUID.randomUUID())); + edge.setTenantId(tenantId); + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + return edge; + } + + private static EntityDataQuery getEdgeTypeQuery(List edgeTypes, String edgeNameFilter, List keyFilters) { + EdgeTypeFilter filter = new EdgeTypeFilter(); + filter.setEdgeTypes(edgeTypes); + filter.setEdgeNameFilter(edgeNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java new file mode 100644 index 0000000000..0d0a623f7a --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityGroupFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntitiesByGroupIdFilterTest extends AbstractEDQTest { + + private UUID deviceId; + private UUID deviceId2; + private UUID deviceId3; + + private UUID groupAId; + private UUID groupBId; + + @Before + public void setUp() { + deviceId = createDevice(customerId, "Lora-1"); + deviceId2 = createDevice(customerId, "Lora-2"); + deviceId3 = createDevice(customerId, "Lora-3"); + + // add device and device 2 to Group A + groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // add device and device 3 to Group B + groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); + createRelation(EntityType.ENTITY_GROUP, groupBId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntitiesOfGroupA() { + // get devices of group A + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, deviceId)); + Assert.assertTrue(checkContains(result, deviceId2)); + + //get devices of non-existing group + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(UUID.randomUUID()), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + //add name filter + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "humidity"); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), List.of(nameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntitiesOfGroupA() { + var result = repository.findEntityDataByQuery(tenantId, new CustomerId(UUID.randomUUID()), RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + } + + @Test + public void testFindCustomerEntitiesWithGroupPermission() { + MergedUserPermissions groupAPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupAPermission, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + + MergedUserPermissions groupBPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(groupBId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + result = repository.findEntityDataByQuery(tenantId, customerId, groupBPermission, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntitiesWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Subcustomer A"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupBId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, + getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupBId), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, deviceId3)); + } + + private static EntityDataQuery getEntitiesByGroupDataQuery(EntityType entityType, EntityGroupId groupId, List keyFilters) { + EntityGroupFilter filter = new EntityGroupFilter(); + filter.setGroupType(entityType); + filter.setEntityGroup(groupId.getId().toString()); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java new file mode 100644 index 0000000000..a6a81abd46 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java @@ -0,0 +1,140 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntitiesByGroupNameFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntitiesByGroupNameFilterTest extends AbstractEDQTest { + + private UUID deviceId; + private UUID deviceId2; + private UUID deviceId3; + + private UUID groupAId; + private UUID groupBId; + + @Before + public void setUp() { + deviceId = createDevice(customerId, "Lora-1"); + deviceId2 = createDevice(customerId, "Lora-2"); + deviceId3 = createDevice(customerId, "Lora-3"); + + // add device and device 2 to Group A + groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // add device and device 3 to Group B + groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); + createRelation(EntityType.ENTITY_GROUP, groupBId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntitiesOfGroupA() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group B", null, null), false); + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntitiesOfGroupA() { + var result = repository.findEntityDataByQuery(tenantId, new CustomerId(UUID.randomUUID()), RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + } + + @Test + public void testFindCustomerEntitiesOfGroupAWithGroupPermission() { + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + } + + @Test + public void testFindGroupWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Subcustomer A"); + UUID subCustomerGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group B"); + UUID deviceId4 = createDevice(subCustomer, "Lora-4"); + createRelation(EntityType.ENTITY_GROUP, subCustomerGroupId, EntityType.DEVICE, deviceId4, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupBId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, + getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group B", null, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + } + + private static EntityDataQuery getEntitiesByGroupNameDataQuery(EntityType entityType, String groupName, EntityId ownerId, List keyFilters) { + EntitiesByGroupNameFilter filter = new EntitiesByGroupNameFilter(); + filter.setGroupType(entityType); + filter.setEntityGroupNameFilter(groupName); + filter.setOwnerId(ownerId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java new file mode 100644 index 0000000000..971b9d52ad --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java @@ -0,0 +1,189 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityGroupListFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class EntityGroupListFilterTest extends AbstractEDQTest { + + private EntityGroup deviceGroup; + private EntityGroup deviceGroup2; + private EntityGroup dashboardGroup; + + @Before + public void setUp() { + deviceGroup = buildEntityGroup(EntityType.DEVICE, "thermostats"); + deviceGroup2 = buildEntityGroup(EntityType.DEVICE, "humidity-sensors"); + dashboardGroup = buildEntityGroup(EntityType.DASHBOARD, "device dashboards"); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup2); + addOrUpdate(EntityType.ENTITY_GROUP, dashboardGroup); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityGroups() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals("thermostats", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE,List.of(deviceGroup.getId().getId().toString(), deviceGroup2.getId().getId().toString()), null), false); + Assert.assertEquals(2, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString()), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + //add name filter + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "humidity"); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString()), Arrays.asList(nameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindTenantEntityGroupsWithGroupPermissionOnly() { + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.DEVICE_GROUP, Set.of(Operation.ALL)), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, null, groupPermission, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityGroups() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + deviceGroup.setOwnerId(customerId); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindCustomerDeviceGroupWithGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + EntityGroup deviceGroup3 = buildEntityGroup(EntityType.DEVICE, "sensors A"); + deviceGroup3.setOwnerId(subCustomer); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup3); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(deviceGroup3.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup3.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup3.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup3.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup3.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindGroupWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + UUID customerGroupId = createGroup(customerId.getId(), EntityType.DEVICE, "customer group"); + UUID subCustomerGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "subcustomer group"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(customerGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(subCustomerGroupId.toString(), customerGroupId.toString()), null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, customerGroupId)); + Assert.assertTrue(checkContains(result, subCustomerGroupId)); + } + + private EntityGroup buildEntityGroup(EntityType entityType, String name) { + EntityGroup entityGroup = new EntityGroup(); + entityGroup.setId(new EntityGroupId(UUID.randomUUID())); + entityGroup.setTenantId(tenantId); + entityGroup.setOwnerId(tenantId); + entityGroup.setName(name); + entityGroup.setType(entityType); + entityGroup.setCreatedTime(42L); + return entityGroup; + } + + private static EntityDataQuery getEntityGroupListDataQuery(EntityType entityType, List ids, List keyFilters) { + EntityGroupListFilter filter = new EntityGroupListFilter(); + filter.setGroupType(entityType); + filter.setEntityGroupList(ids); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java new file mode 100644 index 0000000000..7ad296e946 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java @@ -0,0 +1,186 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityGroupNameFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntityGroupNameFilterTest extends AbstractEDQTest { + + private EntityGroup deviceGroup; + private EntityGroup deviceGroup2; + private EntityGroup dashboardGroup; + + @Before + public void setUp() { + deviceGroup = buildEntityGroup(EntityType.DEVICE, "thermostats"); + deviceGroup2 = buildEntityGroup(EntityType.DEVICE, "thermostats 2"); + dashboardGroup = buildEntityGroup(EntityType.DASHBOARD, "device dashboards"); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup2); + addOrUpdate(EntityType.ENTITY_GROUP, dashboardGroup); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityGroups() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional group = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals(deviceGroup.getName())).findFirst(); + assertThat(group).isPresent(); + var first = group.get(); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermostats 2", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "humidity", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + //add name filter + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "humidity"); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", List.of(nameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityGroups() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + deviceGroup.setOwnerId(customerId); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindCustomerDeviceGroupWithGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + EntityGroup deviceGroup3 = buildEntityGroup(EntityType.DEVICE, "sensors A"); + deviceGroup3.setOwnerId(subCustomer); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup3); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(deviceGroup3.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityGroupNameDataQuery(EntityType.DEVICE,"sensors", null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup3.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup3.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup3.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindGroupWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + UUID customerGroupId = createGroup(customerId.getId(), EntityType.DEVICE, "sensors A"); + UUID subCustomerGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "sensors A"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(customerGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, + getEntityGroupNameDataQuery(EntityType.DEVICE, "sensors A", null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, customerGroupId)); + Assert.assertTrue(checkContains(result, subCustomerGroupId)); + } + + private EntityGroup buildEntityGroup(EntityType entityType, String name) { + EntityGroup entityGroup = new EntityGroup(); + entityGroup.setId(new EntityGroupId(UUID.randomUUID())); + entityGroup.setTenantId(tenantId); + entityGroup.setOwnerId(tenantId); + entityGroup.setName(name); + entityGroup.setType(entityType); + entityGroup.setCreatedTime(42L); + return entityGroup; + } + + private static EntityDataQuery getEntityGroupNameDataQuery(EntityType entityType, String groupName, List keyFilters) { + EntityGroupNameFilter filter = new EntityGroupNameFilter(); + filter.setGroupType(entityType); + filter.setEntityGroupNameFilter(groupName); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java new file mode 100644 index 0000000000..d47252933f --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java @@ -0,0 +1,202 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class EntityListFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + private EntityGroup deviceGroup; + + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE,List.of(device.getId().getId().toString(), device2.getId().getId().toString())), false); + Assert.assertEquals(2, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS,getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + @Test + public void testFindCustomerDeviceWithGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + device.setCustomerId(subCustomer); + addOrUpdate(EntityType.DEVICE, device); + + // add device to customer device group + UUID deviceGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, deviceGroupId, EntityType.DEVICE, device.getUuidId(), RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(deviceGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + @Test + public void testFindCustomerDeviceWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + device.setCustomerId(subCustomer); + addOrUpdate(EntityType.DEVICE, device); + + // add device to customer device group + UUID deviceGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, deviceGroupId, EntityType.DEVICE, device.getUuidId(), RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(deviceGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityListDataQuery(EntityType entityType, List ids) { + EntityListFilter filter = new EntityListFilter(); + filter.setEntityType(entityType); + filter.setEntityList(ids); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "LoRa-"); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java new file mode 100644 index 0000000000..8b17c30f2d --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.UUID; + +public class EntityNameFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("%1"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("L%"), false); + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceNameQuery(String entityNameFilter) { + EntityNameFilter filter = new EntityNameFilter(); + filter.setEntityType(EntityType.DEVICE); + filter.setEntityNameFilter(entityNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java new file mode 100644 index 0000000000..a2b53b1f7d --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntityTypeFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDeviceEntities() { + // find all tenant devices + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("LoRa-1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(device.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(device.getCreatedTime())); + assertThat(first.get().getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()).isEqualTo("enabled"); + + // find all tenant devices with filter by name + KeyFilter keyFilter = getDeviceNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, List.of(keyFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset entities + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDeviceEntities() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityTypeQuery(EntityType entityType, List keyFilters) { + EntityTypeFilter filter = new EntityTypeFilter(); + filter.setEntityType(entityType); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getDeviceNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java new file mode 100644 index 0000000000..b984fe76b9 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java @@ -0,0 +1,221 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class EntityViewSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantEntityViews() { + UUID asset1 = createAsset( "A1"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID deviceView1 = createView("V1"); + UUID deviceView2 = createView("V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of asset A1 + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views with max level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with type "day 1" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("day 1")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + } + + @Test + public void testFindTenantDevicesWithGroupPermissionOnly() { + UUID eg1 = createGroup(EntityType.ENTITY_VIEW, "Group A"); + + UUID asset1 = createAsset( "A1"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID deviceView1 = createView("V1"); + UUID deviceView2 = createView("V2"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, deviceView1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ENTITY_VIEW, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + } + + @Test + public void testFindCustomerDevices() { + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId,"D2"); + UUID deviceView1 = createView(customerId.getId(),"day 1", "V1"); + UUID deviceView2 = createView(customerId.getId(), "day 1", "V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of type "day 1" + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views of type "day 2" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 2")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerEntityViewsWithGroupPermission() { + UUID eg1 = createGroup(EntityType.ENTITY_VIEW, "Group A"); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId,"D2"); + UUID deviceView1 = createView(customerId.getId(),"day 1", "V1"); + UUID deviceView2 = createView(customerId.getId(), "day 1", "V2"); + + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, deviceView1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ENTITY_VIEW, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + } + + @Test + public void testFindCustomerAssetsWithGenericAndGroupPermission() { + CustomerId customerB = new CustomerId(UUID.randomUUID()); + createCustomer(customerB.getId(), customerId.getId(), "Customer B"); + + UUID eg1 = createGroup(customerB.getId(), EntityType.ENTITY_VIEW, "Group A"); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerB.getId(), defaultDeviceProfileId,"D2"); + UUID deviceView1 = createView(customerId.getId(),"day 1", "V1"); + UUID deviceView2 = createView(customerB.getId(), "day 1", "V2"); + + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, deviceView1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views with generic and group permission + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ENTITY_VIEW, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + } + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List entityViewTypes) { + EntityViewSearchQueryFilter filter = new EntityViewSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEntityViewTypes(entityViewTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "V"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java new file mode 100644 index 0000000000..4b086f1ef5 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EntityViewTypeFilterTest extends AbstractEDQTest { + + private EntityView entityView; + private EntityView entityView2; + private EntityView entityView3; + + + @Before + public void setUp() { + entityView = buildEntityView("day 1", "day 1 lora 1 view"); + entityView2 = buildEntityView("day 1", "day 1 lora 2 view"); + entityView3 = buildEntityView("day 2", "day 2 lora 1 view"); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityView() { + // find entity view with type "day 1" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with types "day 1" and "day 2" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Arrays.asList("day 1", "day 2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 2 lora 1 view")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(entityView3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Lora%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 1 lora%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Lora 1 device view" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), "day 1 lora 1 view", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Parking sensor%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 3 lora%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Lora" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "lora" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "lora", false); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityView() { + addOrUpdate(new LatestTsKv(entityView.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + entityView.setCustomerId(customerId); + entityView2.setCustomerId(customerId); + entityView3.setCustomerId(customerId); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private EntityView buildEntityView(String type, String name) { + EntityView entityView = new EntityView(); + entityView.setId(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setType(type); + entityView.setName(name); + entityView.setCreatedTime(42L); + return entityView; + } + + private static EntityDataQuery getEntityViewTypeQuery(List assetTypes, String entityViewNameFilter, List keyFilters) { + EntityViewTypeFilter filter = new EntityViewTypeFilter(); + filter.setEntityViewTypes(assetTypes); + filter.setEntityViewNameFilter(entityViewNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java new file mode 100644 index 0000000000..b0a90d9dce --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java @@ -0,0 +1,233 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; + +public class RelationsQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice("NOT MATCHING D3"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da3, "Contains"); + + PageData relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta2)); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da2)); + } + + @Test + public void testFindTenantDevicesLastLevelOnly() { + UUID root = createAsset("T ROOT"); + + UUID ta1 = createAsset("T A1 NO MORE RELATIONS"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice(customerId, "T D3"); + UUID da4 = createDevice(customerId, "T D4"); // Lvl 4 + + // ROOT --Contains--> A1, A2; A2 --Contains--> D1, D2; D2 --Contains--> D3. + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da2, "Contains"); + createRelation(EntityType.ASSET, da2, EntityType.DEVICE, da3, "Contains"); + createRelation(EntityType.ASSET, da3, EntityType.DEVICE, da4, "Contains"); + + PageData relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 1, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, ta2)); + + relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 2, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da2)); + + relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 3, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da3)); + + relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 4, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da4)); + + } + + @Test + public void testFindTenantDevicesGroupsOnly() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + + PageData relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice(customerId, "T D1"); + UUID da2 = createDevice("T D2"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + + PageData relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerDevicesGroupsOnly() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice(customerId, "T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice(customerId, "T D3"); + + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + createRelation(EntityType.DEVICE, da2, EntityType.DEVICE, da3, "Contains"); + + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + + PageData relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da3)); + + relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da2)); + } + + private PageData filter(EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(RepositoryUtils.ALL_READ_PERMISSIONS, rootId, relationEntityTypeFilters); + } + + private PageData filter(MergedUserPermissions permissions, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(permissions, null, rootId, relationEntityTypeFilters); + } + + private PageData filter(CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, rootId, relationEntityTypeFilters); + } + + private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(permissions, customerId, rootId, 3, false, relationEntityTypeFilters); + } + + private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setRootEntity(rootId); + filter.setFilters(Arrays.asList(relationEntityTypeFilters)); + filter.setDirection(EntitySearchDirection.FROM); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "T"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java new file mode 100644 index 0000000000..42706a796a --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -0,0 +1,434 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate.BooleanOperation; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate.NumericOperation; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class RepositoryUtilsTest { + + private static Stream deviceNameFilters() { + return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet device 123", getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "ra"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "device"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet 123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet "), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet 123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameFilters") + public void testFilterByDeviceName(String deviceName, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream createdTimeFilters() { + return Stream.of(Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 999), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 999), false) + ); + } + + @ParameterizedTest + @MethodSource("createdTimeFilters") + public void testFilterDevicesByCreatedTime(long createdTime, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().createdTime(createdTime).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceNameAndTypeFilter() { + return Stream.of( + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "thermostat")), true), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "thermostat")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false)); + } + + @ParameterizedTest + @MethodSource("deviceNameAndTypeFilter") + public void testFilterByDeviceNameAndDeviceType(String deviceName, String deviceType, List keyFilters, boolean result) { + UUID deviceProfileId = UUID.randomUUID(); + EntityProfileData deviceProfile = new EntityProfileData(deviceProfileId, EntityType.DEVICE_PROFILE); + deviceProfile.setFields(DeviceProfileFields.builder().name(deviceType).build()); + + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).deviceProfileId(deviceProfileId).type(deviceType).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceNameComplexFilters() { + return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "net")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the")), false), + Arguments.of("loranet123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.OR, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameComplexFilters") + public void testFilterByDeviceNameComplexFilters(String deviceName, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceTemperatureFilters() { + return Stream.of(Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.9), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.8), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.9), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 22.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 23.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureFilters") + public void testFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureComplexFilters() { + return Stream.of(Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), true), + Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER, 23.5, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), false), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.OR, NumericOperation.GREATER, 35), true), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.AND, NumericOperation.EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureComplexFilters") + public void testComplexFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceHumidityFilters() { + return Stream.of(Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 59), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 59), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 60), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceHumidityFilters") + public void testFilterByDeviceHumidity(long humidityValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureAndHumidityFilters() { + return Stream.of(Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.EQUAL, 60)), true), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER, 23), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.9), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureAndHumidityFilters") + public void testFilterByDeviceTemperatureAndHumidity(double tempValue, long humidityValue, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceVersionAttributeFilters() { + return Stream.of(Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, true), true), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, false), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, true), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, false), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceVersionAttributeFilters") + public void testFilterByDeviceVersionAttribute(Boolean active, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceActiveAndVersionFilters() { + return Stream.of(Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), true), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceActiveAndVersionFilters") + public void testFilterByActiveAndVersionAttributes(Boolean active, String version, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(1, AttributeScope.CLIENT_SCOPE, new StringDataPoint(System.currentTimeMillis(), version)); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static EdqsFilter getVersionAttributeFilter(StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.CLIENT_ATTRIBUTE, "version", 1); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + + private static EdqsFilter getActiveAttributeFilter(BooleanOperation operation, boolean predicateValue) { + BooleanFilterPredicate filterPredicate = new BooleanFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromBoolean(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.SERVER_ATTRIBUTE, "active", 2); + return new EdqsFilter(key, EntityKeyValueType.BOOLEAN, filterPredicate); + } + + private static EdqsFilter getTemperatureFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("temperature", 5, operation, predicateValue); + } + + private static EdqsFilter getHumidityFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("humidity", 6, operation, predicateValue); + } + + private static EdqsFilter getTimeseriesFilter(String key, Integer keysId, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey newKey = new DataKey(EntityKeyType.TIME_SERIES, key, keysId); + return new EdqsFilter(newKey, EntityKeyValueType.NUMERIC, filterPredicate); + } + + private static EdqsFilter getNameFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("name", operation, predicateValue); + } + + private static EdqsFilter getTypeFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("type", operation, predicateValue); + } + + private static EdqsFilter getStringEntityFieldFilter(String fieldName, StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + private static EdqsFilter getCreatedTimeFilter(NumericOperation operation, double predicateValue) { + return getDatetimeEntityFieldFilter("createdTime", operation, predicateValue); + } + + private static EdqsFilter getDatetimeEntityFieldFilter(String fieldName, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.DATE_TIME, filterPredicate); + } + + private static EdqsFilter getComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2, + ComplexOperation complexOperation2, NumericOperation operation3, double predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexNumericFilterPredicate(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + NumericFilterPredicate filterPredicate2 = new NumericFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromDouble(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2, + ComplexOperation complexOperation2, StringOperation operation3, String predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation3); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue3)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexStringFilterPredicate(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + StringFilterPredicate filterPredicate2 = new StringFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromString(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java new file mode 100644 index 0000000000..8ebbff534d --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.SchedulerEventFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class SchedulerEventFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantSchedulerEvents() { + UUID dashboardId = createDashboard("test dashboard"); + UUID deviceId = createDevice("test device"); + + UUID eventId1 = createSchedulerEvent("Update attributes", new DeviceId(deviceId), "Turn off device"); + UUID eventId2 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate morning report"); + UUID eventId3 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate evening report"); + + // find all scheduler events with type "Generate report" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + Assert.assertTrue(checkContains(result, eventId3)); + + // find all scheduler events for device originator + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId1)); + + // find all scheduler events with name "%morning%" + KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + } + + @Test + public void testFindCustomerEdges() { + UUID dashboardId = createDashboard( "test dashboard"); + UUID deviceId = createDevice("test device"); + + UUID eventId1 = createSchedulerEvent(customerId.getId(), "Update attributes", new DeviceId(deviceId), "Turn off device"); + UUID eventId2 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate morning report"); + UUID eventId3 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate evening report"); + + // find all scheduler events with type "Generate report" + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + Assert.assertTrue(checkContains(result, eventId3)); + + // find all scheduler events for device originator + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId1)); + + // find all scheduler events with name "%morning%" + KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + } + + private static EntityDataQuery getSchedulerEventQuery(String eventType, EntityId entityId, List keyFilters) { + SchedulerEventFilter filter = new SchedulerEventFilter(); + filter.setEventType(eventType); + filter.setOriginator(entityId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java new file mode 100644 index 0000000000..8755191e39 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class SingleEntityFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(1, result.getTotalElements()); + first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindTenantDeviceWithGenericAndGroupPermission() { + UUID deviceId = createDevice(customerId, "LoRa-customer-1"); + UUID deviceId2 = createDevice(customerId, "LoRa-customer-2"); + UUID deviceId3 = createDevice(customerId, "LoRa-customer-3"); + + // add device and device 2 to Group A + UUID groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // add device and device 2 to Group A + UUID groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions genericAndGroupAPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupAPermission, getEntityDataQuery(new DeviceId(deviceId2)), false); + Assert.assertEquals(1, result.getTotalElements()); + QueryResult queryResult = result.getData().get(0); + Assert.assertEquals(deviceId2, queryResult.getEntityId().getId()); + Assert.assertEquals("LoRa-customer-2", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + + // find device without permission + MergedUserPermissions genericAndGroupBPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupBPermission, getEntityDataQuery(new DeviceId(deviceId3)), false); + Assert.assertEquals(1, result.getTotalElements()); + queryResult = result.getData().get(0); + Assert.assertEquals(deviceId3, queryResult.getEntityId().getId()); + Assert.assertEquals("LoRa-customer-3", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { + SingleEntityFilter filter = new SingleEntityFilter(); + filter.setSingleEntity(deviceId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java new file mode 100644 index 0000000000..f0bb4c4e0b --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.UUID; + +public class StateEntityOwnerFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindCustomerDeviceOwner() { + UUID customerId = UUID.randomUUID(); + createCustomer(customerId, null, "Customer A"); + UUID deviceId = createDevice(new CustomerId(customerId), "LoRa-1"); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(deviceId)), false); + + Assert.assertEquals(1, result.getTotalElements()); + var customer = result.getData().get(0); + Assert.assertEquals(customerId, customer.getEntityId().getId()); + Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { + StateEntityOwnerFilter filter = new StateEntityOwnerFilter(); + filter.setSingleEntity(deviceId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/resources/edq-test.properties b/edqs/src/test/resources/edq-test.properties new file mode 100644 index 0000000000..8a041c7407 --- /dev/null +++ b/edqs/src/test/resources/edq-test.properties @@ -0,0 +1,2 @@ +zk.enabled=false +service.type=edqs diff --git a/pom.xml b/pom.xml index e70a57fbbf..42747bae46 100755 --- a/pom.xml +++ b/pom.xml @@ -165,6 +165,8 @@ 1.6.1 2.19.0 9.2.0 + 1.1.10.5 + 9.4.0 @@ -172,6 +174,7 @@ common rule-engine dao + edqs transport ui-ngx tools @@ -1031,6 +1034,11 @@ coap-server ${project.version} + + org.thingsboard.common + edqs + ${project.version} + org.thingsboard.common.script script-api @@ -2279,6 +2287,16 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.xerial.snappy + snappy-java + ${snappy.version} + + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + From c8db304ce6e067dcd00791054ca58fa6b0ad9123 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 30 Jan 2025 12:43:14 +0200 Subject: [PATCH 02/51] Optimized postgres sync (#3208) * optimized postgres sync * Fix consumer stopping when stopWhenRead enabled * Fix NPE on EDQS repartitioning * fixed EntityServiceTest * Fix EDQS yml props --------- Co-authored-by: ViacheslavKlimov --- .../service/edqs/DefaultEdqsService.java | 1 + .../server/service/edqs/EdqsDataLoader.java | 539 ------------------ .../server/service/edqs/EdqsSyncService.java | 102 +++- .../service/edqs/KafkaEdqsSyncService.java | 2 +- .../src/main/resources/thingsboard.yml | 5 +- .../service/entitiy/EntityServiceTest.java | 267 ++++++++- .../common/data/edqs/fields/FieldsUtil.java | 2 + .../server/edqs/processor/EdqsProcessor.java | 30 +- .../ApiUsageStateQueryProcessor.java | 3 +- .../edqs/state/KafkaEdqsStateService.java | 18 +- .../queue/discovery/HashPartitionService.java | 2 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 16 +- .../common/stats/DefaultStatsFactory.java | 2 +- .../server/dao/attributes/AttributesDao.java | 7 +- .../server/dao/relation/RelationDao.java | 4 - .../sql/attributes/AttributeKvRepository.java | 8 + .../dao/sql/attributes/JpaAttributeDao.java | 10 +- .../query/DefaultEntityQueryRepository.java | 4 + .../dao/sql/relation/JpaRelationDao.java | 6 - .../dao/sql/relation/RelationRepository.java | 13 +- .../CachedRedisSqlTimeseriesLatestDao.java | 4 - .../dao/sqlts/SqlTimeseriesLatestDao.java | 4 - .../sqlts/latest/TsKvLatestRepository.java | 8 + .../CassandraBaseTimeseriesLatestDao.java | 4 - .../dao/timeseries/TimeseriesLatestDao.java | 1 - .../edqs/ThingsboardEdqsApplication.java | 10 +- edqs/src/main/resources/edqs.yml | 7 +- 27 files changed, 426 insertions(+), 653 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 7582d036e2..160d4bb565 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -125,6 +125,7 @@ public class DefaultEdqsService implements EdqsService { executor.submit(() -> { try { EdqsSyncState syncState = getSyncState(); + // FIXME: Slavik smart events check if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { processSystemRequest(ToCoreEdqsRequest.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java deleted file mode 100644 index 69a3f6108d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java +++ /dev/null @@ -1,539 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edqs; - -import com.fasterxml.jackson.databind.MappingIterator; -import com.fasterxml.jackson.dataformat.csv.CsvMapper; -import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.ApiUsageState; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceProfileType; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.ObjectType; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.converter.Converter; -import org.thingsboard.server.common.data.converter.ConverterType; -import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edqs.AttributeKv; -import org.thingsboard.server.common.data.edqs.LatestTsKv; -import org.thingsboard.server.common.data.group.EntityGroup; -import org.thingsboard.server.common.data.id.ApiUsageStateId; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.ConverterId; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.EntityGroupId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.IntegrationId; -import org.thingsboard.server.common.data.id.RoleId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.SchedulerEventId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.TenantProfileId; -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.id.WidgetTypeId; -import org.thingsboard.server.common.data.id.WidgetsBundleId; -import org.thingsboard.server.common.data.integration.Integration; -import org.thingsboard.server.common.data.integration.IntegrationType; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.common.data.role.Role; -import org.thingsboard.server.common.data.role.RoleType; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.scheduler.SchedulerEvent; -import org.thingsboard.server.common.data.widget.WidgetType; -import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.common.msg.edqs.EdqsService; -import org.thingsboard.server.edqs.processor.EdqsConverter; - -import java.io.FileReader; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; - -import static org.thingsboard.common.util.JacksonUtil.toJsonNode; - - -@RequiredArgsConstructor -@Slf4j -//@Service -public class EdqsDataLoader { - - private final EdqsService edqsService; - private final EdqsConverter edqsConverter; - - public final static TenantId MAIN = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); - - private final String folder = "/home/viacheslav/Downloads/schwarz"; - - private ExecutorService executor = Executors.newFixedThreadPool(5, ThingsBoardThreadFactory.forName("edqs-publisher")); - -// @AfterStartUp(order = 100) - public void load() throws Exception { - loadCustomers(); - loadDeviceProfile(); - loadDevices(); - loadAssets(); - loadEdges(); - loadEntityViews(); - loadTenants(); - loadUsers(); - loadDashboards(); - loadRuleChains(); - loadWidgetType(); - loadWidgetBundle(); - loadConverters(); - loadIntegrations(); - loadSchedulerEvents(); - loadRoles(); - loadApiUsageStates(); - loadAssetProfile(); - loadEntityGroups(); - loadRelations(); - - loadAttributes(); - loadTs(); - } - - private void loadCustomers() throws Exception { - load("customer.csv", (values) -> { - Customer customer = new Customer(); - customer.setTitle(values.get("title")); - customer.setId(new CustomerId(UUID.fromString(values.get("id")))); - customer.setCreatedTime(Long.parseLong(values.get("created_time"))); - customer.setTenantId(tenantId(values.get("tenant_id"))); - var parentCustomerId = values.get("parent_customer_id"); - if (StringUtils.isNotEmpty(parentCustomerId)) { - customer.setParentCustomerId(new CustomerId(UUID.fromString(parentCustomerId))); - } - edqsService.onUpdate(customer.getTenantId(), customer.getId(), customer); - }); - } - - private void loadDevices() throws Exception { - load("device.csv", (values) -> { - Device device = new Device(); - device.setType(values.get("type")); - device.setName(values.get("name")); - device.setLabel(values.get("label")); - device.setId(new DeviceId(uuid(values.get("id")))); - device.setCreatedTime(parseLong(values.get("created_time"))); - device.setCustomerId(customerId(values.get("customer_id"))); - device.setTenantId(tenantId(values.get("tenant_id"))); - device.setDeviceProfileId(new DeviceProfileId(uuid(values.get("device_profile_id")))); - device.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(device.getTenantId(), device.getId(), device); - }); - } - - private void loadAssets() throws Exception { - load("asset.csv", (values) -> { - Asset asset = new Asset(); - asset.setType(values.get("type")); - asset.setName(values.get("name")); - asset.setLabel(values.get("label")); - asset.setId(new AssetId(uuid(values.get("id")))); - asset.setCreatedTime(parseLong(values.get("created_time"))); - asset.setCustomerId(customerId(values.get("customer_id"))); - asset.setTenantId(tenantId(values.get("tenant_id"))); - asset.setAssetProfileId(new AssetProfileId(uuid(values.get("asset_profile_id")))); - asset.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(asset.getTenantId(), asset.getId(), asset); - }); - } - - private void loadEdges() throws Exception { - load("edge.csv", (values) -> { - Edge edge = new Edge(); - edge.setId(new EdgeId(uuid(values.get("id")))); - edge.setCreatedTime(parseLong(values.get("created_time"))); - edge.setType(values.get("type")); - edge.setName(values.get("name")); - edge.setLabel(values.get("label")); - edge.setCustomerId(customerId(values.get("customer_id"))); - edge.setTenantId(tenantId(values.get("tenant_id"))); - edge.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(edge.getTenantId(), edge.getId(), edge); - }); - } - - private void loadEntityViews() throws Exception { - load("entity_view.csv", (values) -> { - EntityView entityView = new EntityView(); - entityView.setId(new EntityViewId(uuid(values.get("id")))); - entityView.setCreatedTime(parseLong(values.get("created_time"))); - entityView.setType(values.get("type")); - entityView.setName(values.get("name")); - entityView.setCustomerId(customerId(values.get("customer_id"))); - entityView.setTenantId(tenantId(values.get("tenant_id"))); - entityView.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(entityView.getTenantId(), entityView.getId(), entityView); - }); - } - - private void loadTenants() throws Exception { - load("tenant.csv", (values) -> { - Tenant tenant = new Tenant(); - tenant.setId(new TenantId(uuid(values.get("id")))); - tenant.setCreatedTime(parseLong(values.get("created_time"))); - tenant.setEmail(values.get("email")); - tenant.setTitle(values.get("title")); - tenant.setCountry(values.get("country")); - tenant.setState(values.get("state")); - tenant.setCity(values.get("city")); - tenant.setAddress(values.get("address")); - tenant.setAddress2(values.get("address2")); - tenant.setZip(values.get("zip")); - tenant.setPhone(values.get("phone")); - tenant.setRegion(values.get("region")); - tenant.setTenantProfileId(new TenantProfileId(uuid(values.get("tenant_profile_id")))); - tenant.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - edqsService.onUpdate(MAIN, tenant.getId(), tenant); - }); - } - - private void loadUsers() throws Exception { - load("user.csv", (values) -> { - User user = new User(); - user.setId(new UserId(uuid(values.get("id")))); - user.setCreatedTime(parseLong(values.get("created_time"))); - user.setTenantId(tenantId(values.get("tenant_id"))); - user.setFirstName(values.get("first_name")); - user.setLastName(values.get("last_name")); - user.setEmail(values.get("email")); - user.setPhone(values.get("phone")); - user.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(user.getTenantId(), user.getId(), user); - }); - } - - private void loadDashboards() throws Exception { - load("dashboard.csv", (values) -> { - Dashboard dashboard = new Dashboard(); - dashboard.setId(new DashboardId(uuid(values.get("id")))); - dashboard.setCreatedTime(parseLong(values.get("created_time"))); - dashboard.setTenantId(tenantId(values.get("tenant_id"))); - dashboard.setTitle(values.get("title")); - - edqsService.onUpdate(dashboard.getTenantId(), dashboard.getId(), dashboard); - }); - } - - private void loadEntityGroups() throws Exception { - load("entity_group.csv", (values) -> { - EntityGroup entityGroup = new EntityGroup(); - entityGroup.setId(new EntityGroupId(uuid(values.get("id")))); - entityGroup.setCreatedTime(parseLong(values.get("created_time"))); - entityGroup.setName(values.get("name")); - entityGroup.setOwnerId(entityId(values.get("owner_type"), values.get("owner_id"))); - entityGroup.setType(EntityType.valueOf(values.get("type"))); - edqsService.onUpdate(MAIN, entityGroup.getId(), entityGroup); - }); - } - - private void loadRelations() throws Exception { - load("relation.csv", (values) -> { - EntityRelation entityRelation = new EntityRelation(); - entityRelation.setFrom(entityId(values.get("from_type"), values.get("from_id"))); - entityRelation.setTo(entityId(values.get("to_type"), values.get("to_id"))); - entityRelation.setTypeGroup(RelationTypeGroup.valueOf(values.get("relation_type_group"))); - entityRelation.setType(values.get("relation_type")); - edqsService.onUpdate(MAIN, ObjectType.RELATION, entityRelation); - }); - } - - private void loadRuleChains() throws Exception { - load("rule_chain.csv", (values) -> { - RuleChain ruleChain = new RuleChain(); - ruleChain.setId(new RuleChainId(uuid(values.get("id")))); - ruleChain.setCreatedTime(parseLong(values.get("created_time"))); - ruleChain.setName(values.get("name")); - ruleChain.setTenantId(tenantId(values.get("tenant_id"))); - ruleChain.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(ruleChain.getTenantId(), ruleChain.getId(), ruleChain); - }); - } - - private void loadWidgetType() throws Exception { - load("widget_type.csv", (values) -> { - WidgetType widgetType = new WidgetType(); - widgetType.setId(new WidgetTypeId(uuid(values.get("id")))); - widgetType.setCreatedTime(parseLong(values.get("created_time"))); - widgetType.setName(values.get("name")); - widgetType.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(widgetType.getTenantId(), widgetType.getId(), widgetType); - }); - } - - private void loadWidgetBundle() throws Exception { - load("widgets_bundle.csv", (values) -> { - WidgetsBundle widgetsBundle = new WidgetsBundle(); - widgetsBundle.setId(new WidgetsBundleId(uuid(values.get("id")))); - widgetsBundle.setCreatedTime(parseLong(values.get("created_time"))); - widgetsBundle.setTitle(values.get("title")); - widgetsBundle.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle); - }); - } - - private void loadConverters() throws Exception { - load("converter.csv", (values) -> { - Converter converter = new Converter(); - converter.setId(new ConverterId(uuid(values.get("id")))); - converter.setCreatedTime(parseLong(values.get("created_time"))); - converter.setName(values.get("name")); - converter.setType(ConverterType.valueOf(values.get("type"))); - converter.setTenantId(tenantId(values.get("tenant_id"))); - converter.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); - converter.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(converter.getTenantId(), converter.getId(), converter); - }); - } - - private void loadIntegrations() throws Exception { - load("integration.csv", (values) -> { - Integration integration = new Integration(); - integration.setId(new IntegrationId(uuid(values.get("id")))); - integration.setCreatedTime(parseLong(values.get("created_time"))); - integration.setName(values.get("name")); - integration.setType(IntegrationType.valueOf(values.get("type"))); - integration.setTenantId(tenantId(values.get("tenant_id"))); - integration.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); - integration.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(integration.getTenantId(), integration.getId(), integration); - }); - } - - private void loadSchedulerEvents() throws Exception { - load("scheduler_event.csv", (values) -> { - SchedulerEvent schedulerEvent = new SchedulerEvent(); - schedulerEvent.setId(new SchedulerEventId(uuid(values.get("id")))); - schedulerEvent.setCreatedTime(parseLong(values.get("created_time"))); - schedulerEvent.setName(values.get("name")); - schedulerEvent.setType(values.get("type")); - schedulerEvent.setTenantId(tenantId(values.get("tenant_id"))); - schedulerEvent.setConfiguration(toJsonNode(values.get("configuration"))); - schedulerEvent.setSchedule(toJsonNode(values.get("schedule"))); - schedulerEvent.setOriginatorId(entityId(values.get("originator_type"), values.get("originator_id"))); - schedulerEvent.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(schedulerEvent.getTenantId(), schedulerEvent.getId(), schedulerEvent); - }); - } - - private void loadRoles() throws Exception { - load("role.csv", (values) -> { - Role role = new Role(); - role.setId(new RoleId(uuid(values.get("id")))); - role.setCreatedTime(parseLong(values.get("created_time"))); - role.setName(values.get("name")); - role.setType(RoleType.valueOf(values.get("type"))); - role.setTenantId(tenantId(values.get("tenant_id"))); - role.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(role.getTenantId(), role.getId(), role); - }); - } - - private void loadApiUsageStates() throws Exception { - load("api_usage_state.csv", (values) -> { - ApiUsageState apiUsageState = new ApiUsageState(); - apiUsageState.setId(new ApiUsageStateId(uuid(values.get("id")))); - apiUsageState.setCreatedTime(parseLong(values.get("created_time"))); - apiUsageState.setEntityId(entityId(values.get("entity_type"), values.get("entity_id"))); - apiUsageState.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState); - }); - } - - private void loadDeviceProfile() throws Exception { - load("device_profile.csv", (values) -> { - DeviceProfile deviceProfile = new DeviceProfile(); - deviceProfile.setId(new DeviceProfileId(uuid(values.get("id")))); - deviceProfile.setCreatedTime(parseLong(values.get("created_time"))); - deviceProfile.setName(values.get("name")); - deviceProfile.setType(DeviceProfileType.valueOf(values.get("type"))); - deviceProfile.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile); - }); - } - - private void loadAssetProfile() throws Exception { - load("asset_profile.csv", (values) -> { - AssetProfile assetProfile = new AssetProfile(); - assetProfile.setId(new AssetProfileId(uuid(values.get("id")))); - assetProfile.setCreatedTime(parseLong(values.get("created_time"))); - assetProfile.setName(values.get("name")); - assetProfile.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(assetProfile.getTenantId(), assetProfile.getId(), assetProfile); - }); - } - - private void loadAttributes() throws Exception { - load("attribute.csv", (values) -> { - EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("entity_type"), values.get("entity_id")); - long ts = parseLong(values.get("last_update_ts")); - AttributeScope scope = AttributeScope.valueOf(values.get("attribute_type")); - String key = values.get("attribute_key"); - KvEntry kvEntry; - if (StringUtils.isNotEmpty(values.get("bool_v"))) { - kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); - } else if (StringUtils.isNotEmpty(values.get("str_v"))) { - kvEntry = new StringDataEntry(key, values.get("str_v")); - } else if (StringUtils.isNotEmpty(values.get("long_v"))) { - kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); - } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { - kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); - } else if (StringUtils.isNotEmpty(values.get("json_v"))) { - kvEntry = new JsonDataEntry(key, values.get("json_v")); - } else { - kvEntry = new StringDataEntry(key, ""); - } - AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(ts, kvEntry); - AttributeKv attributeKv = new AttributeKv(entityId, scope, attributeKvEntry, 0); - edqsService.onUpdate(MAIN, ObjectType.ATTRIBUTE_KV, attributeKv); - }); - } - - private void loadTs() throws Exception { - load("ts_kv.csv", (values) -> { - var entityTypeStr = values.get("find_entity_type"); - if (StringUtils.isEmpty(entityTypeStr)) { - return; - } - EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("find_entity_type"), values.get("entity_id")); - long ts = parseLong(values.get("ts")); - String key = values.get("key"); - KvEntry kvEntry; - if (StringUtils.isNotEmpty(values.get("bool_v"))) { - kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); - } else if (StringUtils.isNotEmpty(values.get("str_v"))) { - kvEntry = new StringDataEntry(key, values.get("str_v")); - } else if (StringUtils.isNotEmpty(values.get("long_v"))) { - kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); - } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { - kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); - } else if (StringUtils.isNotEmpty(values.get("json_v"))) { - kvEntry = new JsonDataEntry(key, values.get("json_v")); - } else { - kvEntry = new StringDataEntry(key, ""); - } - BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(ts, kvEntry); - edqsService.onUpdate(MAIN, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, 0L)); - }); - } - - private void load(String file, Consumer> function) throws Exception { - Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("loader-" + file)).submit(() -> { - try { - long ts = System.currentTimeMillis(); - CsvSchema schema = CsvSchema.emptySchema().withHeader().withColumnSeparator('|'); - CsvMapper mapper = new CsvMapper(); - MappingIterator> it = mapper - .readerFor(Map.class) - .with(schema) - .readValues(new FileReader(folder + "/" + file)); - - int success = 0; - int failure = 0; - while (it.hasNextValue()) { - Map row = it.nextValue(); - try { - function.accept(row); - success++; - if (success % 1000 == 0) { - log.info("Loaded [{}] from [{}]", success, file); - } - } catch (Exception e) { - log.error("Failed to parse str: [{}]", row, e); - failure++; - } - } - log.info("Loaded [{}] from [{}] in {}ms. Failures {}", success, file, (System.currentTimeMillis() - ts), failure); - } catch (Throwable t) { - log.error("Failed to load data from [{}]", file, t); - } - }); - } - - private static TenantId tenantId(String id) { - return TenantId.fromUUID(UUID.fromString(id)); - } - - private static CustomerId customerId(String id) { - var c = new CustomerId(UUID.fromString(id)); - return c.isNullUid() ? null : c; - } - - private static EntityId entityId(String type, String id) { - return EntityIdFactory.getByTypeAndId(type, id); - } - - private static UUID uuid(String id) { - return UUID.fromString(id); - } - - private static long parseLong(String time) { - return Long.parseLong(time); - } - - private static boolean parseBoolean(String value) { - return Boolean.parseBoolean(value); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 8e12da56e0..bf8e096059 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -32,8 +32,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.attributes.AttributesDao; @@ -41,13 +39,15 @@ import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.entity.EntityDaoRegistry; import org.thingsboard.server.dao.group.EntityGroupDao; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.relation.RelationDao; +import org.thingsboard.server.dao.sql.relation.RelationRepository; +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import java.util.EnumSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -92,11 +92,11 @@ public abstract class EdqsSyncService { @Autowired private KeyDictionaryDao keyDictionaryDao; @Autowired - private RelationDao relationDao; + private RelationRepository relationRepository; @Autowired private EntityGroupDao entityGroupDao; @Autowired - private TimeseriesLatestDao timeseriesLatestDao; + private TsKvLatestRepository tsKvLatestRepository; @Autowired @Lazy private DefaultEdqsService edqsService; @@ -171,7 +171,7 @@ public abstract class EdqsSyncService { private void syncEntityGroups() { log.info("Synchronizing entity groups to EDQS"); long ts = System.currentTimeMillis(); - var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 10000); + var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 30000); for (EntityFields groupFields : entityGroups) { EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId()); if (entityIdInfo != null) { @@ -187,18 +187,43 @@ public abstract class EdqsSyncService { private void syncRelations() { log.info("Synchronizing relations to EDQS"); long ts = System.currentTimeMillis(); - var relations = new PageDataIterable<>(relationDao::findAll, 10000); - for (EntityRelation relation : relations) { - if (relation.getTypeGroup() == RelationTypeGroup.COMMON || relation.getTypeGroup() == RelationTypeGroup.FROM_ENTITY_GROUP) { - EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFrom().getId()); + UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastFromEntityType = ""; + String lastRelationTypeGroup = ""; + String lastRelationType = ""; + UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastToEntityType = ""; + + while (true) { + List batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, + lastRelationType, lastToEntityId, lastToEntityType, 10000); + if (batch.isEmpty()) { + break; + } + processRelationBatch(batch); + + RelationEntity lastRecord = batch.get(batch.size() - 1); + lastFromEntityId = lastRecord.getFromId(); + lastFromEntityType = lastRecord.getFromType(); + lastRelationTypeGroup = lastRecord.getRelationTypeGroup(); + lastRelationType = lastRecord.getRelationType(); + lastToEntityId = lastRecord.getToId(); + lastToEntityType = lastRecord.getToType(); + } + log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processRelationBatch(List relations) { + for (RelationEntity relation : relations) { + if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup()) || (RelationTypeGroup.FROM_ENTITY_GROUP.name().equals(relation.getRelationTypeGroup()))) { + EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); if (entityIdInfo != null) { - process(entityIdInfo.tenantId(), RELATION, relation); + process(entityIdInfo.tenantId(), RELATION, relation.toData()); } else { - log.info("Relation from entity not found: " + relation.getFrom()); + log.info("Relation from entity not found: " + relation.getFromId()); } } } - log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private void loadKeyDictionary() { @@ -214,8 +239,28 @@ public abstract class EdqsSyncService { private void syncAttributes() { log.info("Synchronizing attributes to EDQS"); long ts = System.currentTimeMillis(); - var attributes = new PageDataIterable<>(attributesDao::findAll, 10000); - for (AttributeKvEntity attribute : attributes) { + + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastAttributeType = Integer.MIN_VALUE; + int lastAttributeKey = Integer.MIN_VALUE; + + while (true) { + List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, 10000); + if (batch.isEmpty()) { + break; + } + processAttributeBatch(batch); + + AttributeKvEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getId().getEntityId(); + lastAttributeType = lastRecord.getId().getAttributeType(); + lastAttributeKey = lastRecord.getId().getAttributeKey(); + } + log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processAttributeBatch(List batch) { + for (AttributeKvEntity attribute : batch) { attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); UUID entityId = attribute.getId().getEntityId(); EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); @@ -230,13 +275,29 @@ public abstract class EdqsSyncService { attribute.getVersion()); process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); } - log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private void syncLatestTimeseries() { log.info("Synchronizing latest timeseries to EDQS"); long ts = System.currentTimeMillis(); - var tsKvLatestEntities = new PageDataIterable<>(pageLink -> timeseriesLatestDao.findAllLatest(pageLink), 10000); + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastKey = Integer.MIN_VALUE; + + while (true) { + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); + if (batch.isEmpty()) { + break; + } + processTsKvLatestBatch(batch); + + TsKvLatestEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getEntityId(); + lastKey = lastRecord.getKey(); + } + log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processTsKvLatestBatch(List tsKvLatestEntities) { for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { try { String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); @@ -256,7 +317,6 @@ public abstract class EdqsSyncService { log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); } } - log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private String getStrKeyOrFetchFromDb(int key) { @@ -265,7 +325,9 @@ public abstract class EdqsSyncService { return strKey; } else { strKey = keyDictionaryDao.getKey(key); - keys.putIfAbsent(key, strKey); + if (strKey != null) { + keys.put(key, strKey); + } } return strKey; } diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index d51d87ed01..f4e9a02b45 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -40,7 +40,7 @@ public class KafkaEdqsSyncService extends EdqsSyncService { @Override public boolean isSyncNeeded() { - return kafkaAdmin.isTopicEmpty(EdqsQueue.STATE.getTopic()); + return kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9b182ce3a7..dd0e5948c7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1616,11 +1616,11 @@ queue: # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions - edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions - edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1701,6 +1701,7 @@ queue: local: rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:/tmp/edqs-backup}" partitions: "${TB_EDQS_PARTITIONS:12}" + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index dc29cec3cc..04bad95be7 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -210,7 +210,6 @@ public class EntityServiceTest extends AbstractControllerTest { countByQueryAndCheck(countQuery, 0); } - @Test public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { List assets = new ArrayList<>(); @@ -463,6 +462,7 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } + // fails for sql implementation until we fix the issue with the relation query @Test public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { List buildings = new ArrayList<>(); @@ -1443,18 +1443,29 @@ public class EntityServiceTest extends AbstractControllerTest { String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); + // find by customer user with generic permission + MergedUserPermissions mergedGenericPermission = new MergedUserPermissions(Map.of(Resource.DEVICE, Set.of(Operation.READ)), Collections.emptyMap()); + PageData customerResults = findByQueryAndCheck(customerId, mergedGenericPermission, query, 1); + + String cutomerDeviceName = customerResults.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(cutomerDeviceName).isEqualTo(customerDevices.get(0).getName()); + + // find by customer user with group permission + MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(customerDeviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); + PageData result2 = findByQueryAndCheck(customerId, mergedGroupOnlyPermission, query, 1); + + String resultDeviceName2 = result2.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(resultDeviceName2).isEqualTo(customerDevices.get(0).getName()); + // try to find tenant device by customer user SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); - PageData customerResults2 = entityService.findEntityDataByQuery(tenantId, customerId, customerQuery2); - - assertEquals(0, customerResults2.getTotalElements()); + findByQueryAndCheck(customerId, mergedGenericPermission, customerQuery2, 0); // find by tenant user with group permission - PageData results3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + PageData results3 = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), mergedGroupOnlyPermission, query, 1); - assertEquals(1, results3.getTotalElements()); String deviceName3 = results3.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName3).isEqualTo(customerDevices.get(0).getName()); } @@ -1488,16 +1499,14 @@ public class EntityServiceTest extends AbstractControllerTest { // find by customer user with generic permissions apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); - PageData customerResult = entityService.findEntityDataByQuery(tenantId, customerId, query); + PageData customerResult = findByQueryAndCheck(customerId, query, 1); - assertEquals(1, customerResult.getTotalElements()); String customerResultName = customerResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(customerResultName).isEqualTo(TEST_CUSTOMER_NAME); // find by tenant user with customerId filter apiUsageStateFilter.setCustomerId(customerId); - PageData tenantResult = searchEntities(query); - assertEquals(1, tenantResult.getTotalElements()); + PageData tenantResult = findByQueryAndCheck(query, 1); String tenantResultName = tenantResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); } @@ -1625,6 +1634,231 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } + @Test + public void testFindEntityDataByRelationQuery_blobEntity_customerLevel() { + final int deviceCnt = 2; + final int relationsCnt = 3; + final int blobEntitiesCnt = deviceCnt * relationsCnt; + + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("Customer Relation Query"); + customer = customerService.saveCustomer(customer); + + List devices = new ArrayList<>(); + for (int i = 0; i < deviceCnt; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device relation query " + i); + device.setCustomerId(customer.getId()); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + } + + List blobEntities = new ArrayList<>(); + for (int i = 0; i < blobEntitiesCnt; i++) { + BlobEntity blobEntity = new BlobEntity(); + blobEntity.setName("Blob relation query " + i); + blobEntity.setTenantId(tenantId); + blobEntity.setContentType("image/png"); + blobEntity.setData(ByteBuffer.allocate(1024)); + blobEntity.setCustomerId(customer.getId()); + blobEntity.setType("Report"); + blobEntities.add(blobEntityService.saveBlobEntity(blobEntity)); + } + + for (int i = 0; i < deviceCnt; i++) { + for (int j = 0; j < relationsCnt; j++) { + EntityRelation relationEntity = new EntityRelation(); + relationEntity.setFrom(devices.get(i).getId()); + relationEntity.setTo(blobEntities.get(j + (i * relationsCnt)).getId()); + relationEntity.setTypeGroup(RelationTypeGroup.COMMON); + relationEntity.setType("fileAttached"); + relationService.saveRelation(tenantId, relationEntity); + } + } + + MergedUserPermissions mergedUserPermissions = new MergedUserPermissions(Map.of(ALL, Set.of(Operation.ALL)), Collections.emptyMap()); + + RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("fileAttached", Collections.singletonList(EntityType.BLOB_ENTITY)); + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); + filter.setDirection(EntitySearchDirection.FROM); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + + for (Device device : devices) { + filter.setRootEntity(device.getId()); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + findByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); + countByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); + /* + In order to be careful with updating Relation Query while adding new Entity Type, + this checkup will help to find place, where you could check the correctness of building query + */ + Assert.assertEquals(38, EntityType.values().length); + } + } + + @Test + public void testFindEntitiesByRelationEntityTypeFilterWithTenantGroupPermission() { + final int assetCount = 2; + final int relationsCnt = 4; + final int deviceEntitiesCnt = assetCount * relationsCnt; + + EntityGroup deviceGroup = new EntityGroup(); + deviceGroup.setName("Device Tenant Level Group"); + deviceGroup.setOwnerId(tenantId); + deviceGroup.setTenantId(tenantId); + deviceGroup.setType(EntityType.DEVICE); + deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); + + List assets = new ArrayList<>(); + for (int i = 0; i < assetCount; i++) { + Asset building = new Asset(); + building.setTenantId(tenantId); + building.setName("Building _" + i); + building.setType("building"); + building = assetService.saveAsset(building); + assets.add(building); + } + + List devices = new ArrayList<>(); + for (int i = 0; i < deviceEntitiesCnt; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Test device " + i); + device.setType("default"); + Device savedDevice = deviceService.saveDevice(device); + devices.add(savedDevice); + if (i % 2 == 0) { + entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); + } + } + + for (int i = 0; i < assetCount; i++) { + for (int j = 0; j < relationsCnt; j++) { + EntityRelation relationEntity = new EntityRelation(); + relationEntity.setFrom(assets.get(i).getId()); + relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); + relationEntity.setTypeGroup(RelationTypeGroup.COMMON); + relationEntity.setType("contains"); + relationService.saveRelation(tenantId, relationEntity); + } + } + + MergedUserPermissions groupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), + Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); + + RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); + filter.setDirection(EntitySearchDirection.FROM); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); + + for (Asset asset : assets) { + filter.setRootEntity(asset.getId()); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); + countByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); + } + } + + @Test + public void testFindEntitiesWithRelationEntityTypeFilterByCustomerUser() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("Customer Relation Query"); + customer = customerService.saveCustomer(customer); + + final int assetCount = 2; + final int relationsCnt = 4; + final int deviceEntitiesCnt = assetCount * relationsCnt; + + EntityGroup deviceGroup = new EntityGroup(); + deviceGroup.setName("Device Tenant Level Group"); + deviceGroup.setOwnerId(customer.getId()); + deviceGroup.setTenantId(tenantId); + deviceGroup.setType(EntityType.DEVICE); + deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); + + List assets = new ArrayList<>(); + for (int i = 0; i < assetCount; i++) { + Asset building = new Asset(); + building.setTenantId(tenantId); + building.setCustomerId(customer.getId()); + building.setName("Building _" + i); + building.setType("building"); + building = assetService.saveAsset(building); + assets.add(building); + } + + List devices = new ArrayList<>(); + for (int i = 0; i < deviceEntitiesCnt; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customer.getId()); + device.setName("Test device " + i); + device.setType("default"); + Device savedDevice = deviceService.saveDevice(device); + devices.add(savedDevice); + if (i % 2 == 0) { + entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); + } + } + + for (int i = 0; i < assetCount; i++) { + for (int j = 0; j < relationsCnt; j++) { + EntityRelation relationEntity = new EntityRelation(); + relationEntity.setFrom(assets.get(i).getId()); + relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); + relationEntity.setTypeGroup(RelationTypeGroup.COMMON); + relationEntity.setType("contains"); + relationService.saveRelation(tenantId, relationEntity); + } + } + + MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + MergedUserPermissions mergedGenericOnlyPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.emptyMap()); + MergedUserPermissions mergedGenericAndGroupPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); + filter.setDirection(EntitySearchDirection.FROM); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + + for (Asset asset : assets) { + filter.setRootEntity(asset.getId()); + + //check by user with generic permission + PageData relationsResult = findByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); + countByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); + + //check by user with generic and group permission + PageData relationsResult1 = findByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); + countByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); + + //check by other customer user with group only permission + PageData relationsResult2 = findByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); + long relationsResultCnt2 = countByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); + + Assert.assertEquals(relationsCnt / 2, relationsResult2.getData().size()); + Assert.assertEquals(relationsCnt / 2, relationsResultCnt2); + + //check by other customer user with generic and group only permission + PageData relationsResult3 = findByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); + long relationsResultCnt3 = countByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); + + Assert.assertEquals(relationsCnt / 2, relationsResult3.getData().size()); + Assert.assertEquals(relationsCnt / 2, relationsResultCnt3); + } + } + @Test public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { @@ -2422,7 +2656,7 @@ public class EntityServiceTest extends AbstractControllerTest { return timeseriesService.save(tenantId, entityId, timeseries); } - private void createMultiRootHierarchy(List buildings, List apartments, + protected void createMultiRootHierarchy(List buildings, List apartments, Map> entityNameByTypeMap, Map childParentRelationMap) throws InterruptedException { for (int k = 0; k < 3; k++) { @@ -2507,7 +2741,7 @@ public class EntityServiceTest extends AbstractControllerTest { entityView.setEndTimeMs(256); entityView.setExternalId(new EntityViewId(UUID.randomUUID())); entityView.setAdditionalInfo(JacksonUtil.newObjectNode().put("test", "test")); - entityView = entityViewDao.save(tenantId, entityView); + entityView = entityViewService.saveEntityView(entityView); EntityViewTypeFilter entityViewTypeFilter = new EntityViewTypeFilter(); entityViewTypeFilter.setEntityViewNameFilter("test"); @@ -2518,21 +2752,18 @@ public class EntityServiceTest extends AbstractControllerTest { ); EntityDataQuery query = new EntityDataQuery(entityViewTypeFilter, pageLink, entityFields, Collections.emptyList(), null); - PageData relationsResult = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); - assertThat(relationsResult.getData()).hasSize(1); + PageData relationsResult = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 1); assertThat(relationsResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).isEqualTo(entityView.getName()); // find with non existing name entityViewTypeFilter.setEntityViewNameFilter("non-existing"); - PageData relationsResult2 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); - assertThat(relationsResult2.getData()).hasSize(0); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); // find with non existing type entityViewTypeFilter.setEntityViewNameFilter(null); entityViewTypeFilter.setEntityViewTypes(Collections.singletonList("non-existing")); - PageData relationsResult3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); - assertThat(relationsResult3.getData()).hasSize(0); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); } private PageData findByQuery(EntityDataQuery query) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index 5a0db392ef..f1cf51bf8b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -275,6 +276,7 @@ public class FieldsUtil { return ApiUsageStateFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) + .customerId(entity.getEntityId().getEntityType() == EntityType.CUSTOMER ? entity.getEntityId().getId() : null) .entityId(entity.getEntityId()) .transportState(entity.getTransportState()) .dbStorageState(entity.getDbStorageState()) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 5383311e1f..24526205d8 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -42,10 +42,11 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.repo.EdqRepository; +import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.util.EdqsPartitionService; import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos; @@ -70,6 +71,7 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @EdqsComponent @@ -96,6 +98,8 @@ public class EdqsProcessor implements TbQueueHandler, private final VersionsStore versionsStore = new VersionsStore(); + private final AtomicInteger counter = new AtomicInteger(); // FIXME: TMP + @PostConstruct private void init() { consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); @@ -152,15 +156,17 @@ public class EdqsProcessor implements TbQueueHandler, responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); - Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() - .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); - if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { - log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + if (CollectionsUtil.isNotEmpty(oldPartitions)) { + Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() + .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); + if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + } + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId); + return partition != null && removedPartitions.contains(partition); + }); } - repository.clearIf(tenantId -> { - Integer partition = partitionService.resolvePartition(tenantId); - return partition != null && removedPartitions.contains(partition); - }); } catch (Throwable t) { log.error("Failed to handle partition change event {}", event, t); } @@ -221,7 +227,11 @@ public class EdqsProcessor implements TbQueueHandler, } EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray()); - log.info("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + log.debug("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + int count = counter.incrementAndGet(); + if (count % 100000 == 0) { + log.info("Processed {} events", count); + } EdqsEvent event = EdqsEvent.builder() .tenantId(tenantId) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java index c2821161c7..7dbda46068 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java @@ -76,8 +76,7 @@ public class ApiUsageStateQueryProcessor extends AbstractSingleEntityTypeQueryPr @Override protected boolean matches(EntityData ed) { ApiUsageStateFields entityFields = (ApiUsageStateFields) ed.getFields(); - return super.matches(ed) && (filter.getCustomerId() != null ? entityFields.getEntityId().equals(filter.getCustomerId()) : - entityFields.getEntityId().equals(repository.getTenantId())); + return super.matches(ed) && (filter.getCustomerId() == null || filter.getCustomerId().equals(entityFields.getEntityId())); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index e8e8ee610e..04ef73faa1 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -71,7 +71,8 @@ public class KafkaEdqsStateService implements EdqsStateService { private ScheduledExecutorService scheduler; private final VersionsStore versionsStore = new VersionsStore(); - private final AtomicInteger restoredCount = new AtomicInteger(); + private final AtomicInteger stateReadCount = new AtomicInteger(); + private final AtomicInteger eventsReadCount = new AtomicInteger(); @PostConstruct private void init() { @@ -79,7 +80,7 @@ public class KafkaEdqsStateService implements EdqsStateService { mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); - stateConsumer = MainQueueConsumerManager., QueueConfig>builder() + stateConsumer = MainQueueConsumerManager., QueueConfig>builder() // FIXME Slavik: if topic is empty .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) .config(QueueConfig.of(true, config.getPollInterval())) .msgPackProcessor((msgs, consumer, config) -> { @@ -88,8 +89,8 @@ public class KafkaEdqsStateService implements EdqsStateService { ToEdqsMsg msg = queueMsg.getValue(); log.trace("Processing message: {}", msg); edqsProcessor.process(msg, EdqsQueue.STATE); - if (restoredCount.incrementAndGet() % 1000 == 0) { - log.info("Processed {} msgs", restoredCount.get()); + if (stateReadCount.incrementAndGet() % 100000 == 0) { + log.info("[state] Processed {} msgs", stateReadCount.get()); } } catch (Throwable t) { log.error("Failed to process message: {}", queueMsg, t); @@ -103,7 +104,7 @@ public class KafkaEdqsStateService implements EdqsStateService { .scheduler(scheduler) .build(); - eventsConsumer = QueueConsumerManager.>builder() + eventsConsumer = QueueConsumerManager.>builder() // FIXME Slavik writes to the state while we read it, slows down the start .name("edqs-events-to-backup-consumer") .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer) -> { @@ -115,6 +116,9 @@ public class KafkaEdqsStateService implements EdqsStateService { if (msg.hasEventMsg()) { EdqsEventMsg eventMsg = msg.getEventMsg(); String key = eventMsg.getKey(); + if (eventsReadCount.incrementAndGet() % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", eventsReadCount.get()); + } if (eventMsg.hasVersion()) { if (!versionsStore.isNew(key, eventMsg.getVersion())) { return; @@ -153,14 +157,14 @@ public class KafkaEdqsStateService implements EdqsStateService { @Override public void restore(Set partitions) { - restoredCount.set(0); + stateReadCount.set(0); //TODO Slavik: do not support remote mode in monolith setup long startTs = System.currentTimeMillis(); log.info("Restore started for partitions {}", partitions.stream().map(tpi -> tpi.getPartition().orElse(-1)).sorted().toList()); stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update stateConsumer.awaitStop(0); // consumers should stop on their own because EdqsQueue.STATE.stopWhenRead is true, we just need to wait - log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), restoredCount.get()); + log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), stateReadCount.get()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 165a54b300..ea9ba3f245 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -482,7 +482,7 @@ public class HashPartitionService implements PartitionService { private Set toTpiList(QueueKey queueKey, List partitions) { if (partitions == null) { - return null; + return Collections.emptySet(); } return partitions.stream() .map(partition -> buildTopicPartitionInfo(queueKey, partition)) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 167fbb1751..47c776c742 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -144,10 +144,11 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + List> recordList; if (records.isEmpty()) { - return Collections.emptyList(); + recordList = Collections.emptyList(); } else { - List> recordList = new ArrayList<>(256); + recordList = new ArrayList<>(256); records.forEach(record -> { recordList.add(record); if (stopWhenRead) { @@ -163,17 +164,18 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue } } }); - if (endOffsets != null && endOffsets.isEmpty()) { - log.info("Reached end offsets for {}, stopping consumer", consumer.assignment()); - stop(); - } - return recordList; } + if (stopWhenRead && endOffsets.isEmpty()) { + log.info("Reached end offset for {}, stopping consumer", consumer.assignment()); + stop(); + } + return recordList; } private void onPartitionsAssigned() { if (stopWhenRead) { endOffsets = consumer.endOffsets(consumer.assignment()).entrySet().stream() + .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); } } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index ca97792186..2fc7970fc9 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -38,7 +38,7 @@ public class DefaultStatsFactory implements StatsFactory { private static final Counter STUB_COUNTER = new StubCounter(); - @Autowired + @Autowired(required = false) // FIXME Slavik !!! private MeterRegistry meterRegistry; @Value("${metrics.enabled:false}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 2c37788173..6678be6009 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -22,14 +22,13 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.UUID; /** * @author Andrew Shvayka @@ -42,14 +41,14 @@ public interface AttributesDao { List findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope); - PageData findAll(PageLink pageLink); - ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute); List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); List>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); + List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize); + List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index d29b57c5c3..d668f51dbd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -18,8 +18,6 @@ package org.thingsboard.server.dao.relation; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -43,8 +41,6 @@ public interface RelationDao { List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); - PageData findAll(PageLink pageLink); - ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index 90f49c57fd..de5651be00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -59,4 +59,12 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByEntityIdsAndAttributeType(@Param("entityIds") List entityIds, @Param("attributeType") int attributeType); + + @Query(value = "SELECT attribute_key, attribute_type, entity_id, bool_v, dbl_v, json_v, last_update_ts, long_v, str_v, version FROM attribute_kv WHERE (entity_id, attribute_type, attribute_key) > " + + "(:entityId, :attributeType, :attributeKey) ORDER BY entity_id, attribute_type, attribute_key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("attributeType") int attributeType, + @Param("attributeKey") int attributeKey, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index b344a0ca41..db166c44b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AttributeScope; @@ -31,8 +30,6 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; @@ -51,8 +48,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -157,9 +154,8 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public PageData findAll(PageLink pageLink) { - Page attributes = attributeKvRepository.findAll(DaoUtil.toPageable(pageLink)); - return DaoUtil.pageToPageData(attributes); + public List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize) { + return attributeKvRepository.findNextBatch(entityId, attributeType, attributeKey, batchSize); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 31443f7f47..c54b5561ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -693,6 +693,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE + ", " + SELECT_ADDITIONAL_INFO + (entityFilter.isMultiRoot() ? (", " + SELECT_RELATED_PARENT_ID) : "") + ", entity.entity_type as entity_type"; + /* + * FIXME: + * target entities are duplicated in result list, if search direction is TO and multiple relations are references to target entity + * */ String from = getQueryTemplate(entityFilter.getDirection(), entityFilter.isMultiRoot()); if (entityFilter.isMultiRoot()) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 1df7115518..118c3c1bcc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -24,8 +24,6 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -129,10 +127,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public PageData findAll(PageLink pageLink) { - return DaoUtil.toPageData(relationRepository.findAll(DaoUtil.toPageable(pageLink))); - } @Override public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index d945f1f640..3c69ae095f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -85,6 +85,15 @@ public interface RelationRepository @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType and r.relationTypeGroup in :relationTypeGroups") void deleteByFromIdAndFromTypeAndRelationTypeGroupIn(@Param("fromId") UUID fromId, @Param("fromType") String fromType, @Param("relationTypeGroups") List relationTypeGroups); - @Query("SELECT e FROM RelationEntity e ORDER BY e.fromId, e.fromType, e.toId, e.toType, e.relationType, e.relationTypeGroup") - Page findAll(Pageable pageable); + @Query(value = "SELECT from_id, from_type, relation_type_group, relation_type, to_id, to_type, additional_info, version FROM relation" + + " WHERE (from_id, from_type, relation_type_group, relation_type, to_id, to_type) > " + + "(:fromId, :fromType, :relationTypeGroup, :relationType, :toId, :toType) ORDER BY " + + "from_id, from_type, relation_type_group, relation_type, to_id, to_type LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("toId") UUID toId, + @Param("toType") String toType, + @Param("batchSize") int batchSize); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index 0e9cebd150..e6175409de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -171,9 +171,5 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } - @Override - public PageData findAllLatest(PageLink pageLink) { - return null; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index fd1b180eef..dde89e107d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -188,10 +188,6 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); } - @Override - public PageData findAllLatest(PageLink pageLink) { - return DaoUtil.pageToPageData(tsKvLatestRepository.findAll(DaoUtil.toPageable(pageLink, "entityId", "key"))); - } private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 833ffd185e..2c97ba30ba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; @@ -46,4 +47,11 @@ public interface TsKvLatestRepository extends JpaRepository findAll(Pageable pageable); + + @Query(value = "SELECT entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version FROM ts_kv_latest WHERE (entity_id, key) > " + + "(:entityId, :key) ORDER BY entity_id, key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("key") int key, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 99bf57a1db..6f00437672 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -103,10 +103,6 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } - @Override - public PageData findAllLatest(PageLink pageLink) { - return null; - } @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 1372f2f5d8..b2d7c0eff5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -54,5 +54,4 @@ public interface TimeseriesLatestDao { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); - PageData findAllLatest(PageLink pageLink); } diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java index 1157f74de1..7fed88b0c3 100644 --- a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java +++ b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java @@ -30,7 +30,7 @@ import java.util.Arrays; @EnableAsync @EnableScheduling @ComponentScan({"org.thingsboard.server.edqs", "org.thingsboard.server.queue.edqs", "org.thingsboard.server.queue.discovery", "org.thingsboard.server.queue.kafka", - "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment"}) + "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment", "org.thingsboard.server.common.stats"}) @Slf4j public class ThingsboardEdqsApplication { @@ -42,8 +42,8 @@ public class ThingsboardEdqsApplication { } // @Bean - public ApplicationRunner runner(CSVLoader loader, EdqRepository edqRepository) { - return args -> { +// public ApplicationRunner runner(CSVLoader loader, EdqRepository edqRepository) { +// return args -> { // long startTs = System.currentTimeMillis(); // var loader = new TenantRepoLoader(new TenantRepo(TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")))); // loader.load(); @@ -103,8 +103,8 @@ public class ThingsboardEdqsApplication { // }); // Thread.sleep(5000); // } - }; - } +// }; +// } private static String[] updateArguments(String[] args) { if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index f5c83178c3..143a42fb68 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -52,15 +52,14 @@ queue: # For debug level print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" edqs: - enabled: "${TB_EDQS_ENABLED:true}" mode: "${TB_EDQS_MODE:local}" partitions: "${TB_EDQS_PARTITIONS:12}" + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" - partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data kafka: # Kafka Bootstrap nodes in "host:port" format bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" @@ -171,11 +170,11 @@ queue: # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions - edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions - edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" From 0e9d0b3ba3b456b71fd732a7504d714ac0abc58c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 3 Feb 2025 18:27:36 +0200 Subject: [PATCH 03/51] fixed EntityServiceTest --- .../server/service/edqs/EdqsSyncService.java | 95 ++----- .../EdqsEntityQueryControllerTest.java | 10 +- .../controller/EntityQueryControllerTest.java | 36 +-- .../entitiy/EdqsEntityServiceTest.java | 11 +- .../service/entitiy/EntityServiceTest.java | 244 +----------------- .../server/common/data/ObjectType.java | 17 +- .../data/edqs/fields/DashboardFields.java | 32 ++- .../common/data/edqs/fields/EntityFields.java | 6 + .../data/edqs/fields/EntityViewFields.java | 7 + .../common/data/edqs/fields/FieldsUtil.java | 2 - .../common/data/edqs/query/EdqsRequest.java | 4 - .../common/data/edqs/query/QueryResult.java | 4 +- .../common/data/permission/QueryContext.java | 3 + .../server/common/data/queue/QueueStats.java | 8 +- common/edqs/pom.xml | 4 +- .../server/edqs/data/ApiUsageStateData.java | 2 +- .../server/edqs/data/AssetData.java | 2 +- .../server/edqs/data/BaseEntityData.java | 2 +- .../server/edqs/data/CustomerData.java | 2 +- .../server/edqs/data/DeviceData.java | 2 +- .../server/edqs/data/EntityData.java | 2 +- .../server/edqs/data/EntityGroupData.java | 58 ----- .../server/edqs/data/EntityProfileData.java | 2 +- .../server/edqs/data/GenericData.java | 2 +- .../server/edqs/data/ProfileAwareData.java | 2 +- .../server/edqs/data/RelationData.java | 2 +- .../server/edqs/data/RelationInfo.java | 2 +- .../server/edqs/data/RelationsRepo.java | 2 +- .../server/edqs/data/TenantData.java | 2 +- .../edqs/data/dp/AbstractDataPoint.java | 2 +- .../server/edqs/data/dp/BoolDataPoint.java | 2 +- .../edqs/data/dp/CompressedJsonDataPoint.java | 2 +- .../data/dp/CompressedStringDataPoint.java | 2 +- .../server/edqs/data/dp/DataPoint.java | 2 +- .../server/edqs/data/dp/DoubleDataPoint.java | 2 +- .../server/edqs/data/dp/JsonDataPoint.java | 2 +- .../server/edqs/data/dp/LongDataPoint.java | 2 +- .../server/edqs/data/dp/StringDataPoint.java | 2 +- .../server/edqs/load/TenantRepoLoader.java | 2 +- .../server/edqs/processor/EdqsConverter.java | 2 +- .../server/edqs/processor/EdqsProcessor.java | 9 +- .../server/edqs/processor/EdqsProducer.java | 2 +- .../server/edqs/query/DataKey.java | 2 +- .../server/edqs/query/EdqsCountQuery.java | 2 +- .../server/edqs/query/EdqsDataQuery.java | 2 +- .../server/edqs/query/EdqsFilter.java | 2 +- .../server/edqs/query/EdqsQuery.java | 2 +- .../server/edqs/query/SortableEntityData.java | 4 +- .../AbstractEntityGroupQueryProcessor.java | 75 ------ ...stractEntityProfileNameQueryProcessor.java | 2 +- .../AbstractEntityProfileQueryProcessor.java | 2 +- .../AbstractEntitySearchQueryProcessor.java | 4 +- .../processor/AbstractQueryProcessor.java | 71 +---- .../AbstractRelationQueryProcessor.java | 111 +------- .../AbstractSimpleQueryProcessor.java | 53 +--- ...bstractSingleEntityTypeQueryProcessor.java | 116 +-------- .../ApiUsageStateQueryProcessor.java | 35 +-- .../processor/AssetSearchQueryProcessor.java | 2 +- .../processor/AssetTypeQueryProcessor.java | 2 +- .../query/processor/CombinedPermissions.java | 25 -- .../processor/DeviceSearchQueryProcessor.java | 2 +- .../processor/DeviceTypeQueryProcessor.java | 2 +- .../processor/EdgeTypeQueryProcessor.java | 2 +- .../EdgeTypeSearchQueryProcessor.java | 2 +- .../EntitiesByGroupNameQueryProcessor.java | 121 --------- .../EntitiesByGroupQueryProcessor.java | 96 ------- .../EntityGroupListQueryProcessor.java | 84 ------ .../EntityGroupNameQueryProcessor.java | 83 ------ .../processor/EntityListQueryProcessor.java | 34 +-- .../processor/EntityNameQueryProcessor.java | 2 +- .../query/processor/EntityQueryProcessor.java | 2 +- .../EntityQueryProcessorFactory.java | 8 +- .../processor/EntityTypeQueryProcessor.java | 2 +- .../EntityViewSearchQueryProcessor.java | 2 +- .../EntityViewTypeQueryProcessor.java | 2 +- .../query/processor/GroupPermissions.java | 29 --- .../edqs/query/processor/Permissions.java | 24 -- .../processor/RelationQueryPermissions.java | 33 --- .../processor/RelationQueryProcessor.java | 2 +- .../SchedulerEventQueryProcessor.java | 37 --- .../processor/SingleEntityQueryProcessor.java | 36 +-- .../StateEntityOwnerQueryProcessor.java | 91 ------- .../server/edqs/repo/EdqRepository.java | 7 +- .../edqs/repo/InMemoryEdqRepository.java | 22 +- .../server/edqs/repo/KeyDictionary.java | 2 +- .../server/edqs/repo/TbBytePool.java | 2 +- .../server/edqs/repo/TbStringPool.java | 2 +- .../server/edqs/repo/TenantRepo.java | 142 ++-------- .../server/edqs/state/EdqsStateService.java | 2 +- .../edqs/state/KafkaEdqsStateService.java | 2 +- .../edqs/state/LocalEdqsStateService.java | 2 +- .../server/edqs/stats/EdqsStatsService.java | 2 +- .../edqs/util/EdqsPartitionService.java | 2 +- .../server/edqs/util/EdqsRocksDb.java | 2 +- .../server/edqs/util/RepositoryUtils.java | 39 +-- .../server/edqs/util/TbRocksDb.java | 2 +- .../server/edqs/util/VersionsStore.java | 2 +- .../server/queue/discovery/QueueKey.java | 6 + .../java/org/thingsboard/server/dao/Dao.java | 5 +- .../dao/customer/CustomerServiceImpl.java | 2 +- .../server/dao/entity/BaseEntityService.java | 25 +- .../dao/sql/asset/AssetProfileRepository.java | 5 +- .../server/dao/sql/asset/AssetRepository.java | 5 +- .../server/dao/sql/asset/JpaAssetDao.java | 5 +- .../dao/sql/asset/JpaAssetProfileDao.java | 5 +- .../dao/sql/customer/CustomerRepository.java | 9 +- .../dao/sql/customer/JpaCustomerDao.java | 6 +- .../sql/dashboard/DashboardRepository.java | 6 +- .../dao/sql/dashboard/JpaDashboardDao.java | 5 +- .../sql/device/DeviceProfileRepository.java | 6 +- .../dao/sql/device/DeviceRepository.java | 5 +- .../server/dao/sql/device/JpaDeviceDao.java | 5 +- .../dao/sql/device/JpaDeviceProfileDao.java | 5 +- .../server/dao/sql/edge/EdgeRepository.java | 6 +- .../server/dao/sql/edge/JpaEdgeDao.java | 5 +- .../sql/entityview/EntityViewRepository.java | 7 +- .../dao/sql/entityview/JpaEntityViewDao.java | 7 +- .../query/DefaultAlarmQueryRepository.java | 3 - .../dao/sql/queue/JpaQueueStatsDao.java | 5 +- .../dao/sql/queue/QueueStatsRepository.java | 5 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 6 +- .../dao/sql/rule/RuleChainRepository.java | 5 +- .../server/dao/sql/tenant/JpaTenantDao.java | 11 +- .../dao/sql/tenant/JpaTenantProfileDao.java | 5 +- .../sql/tenant/TenantProfileRepository.java | 5 +- .../dao/sql/tenant/TenantRepository.java | 5 +- .../usagerecord/ApiUsageStateRepository.java | 8 +- .../sql/usagerecord/JpaApiUsageStateDao.java | 6 +- .../server/dao/sql/user/JpaUserDao.java | 5 +- .../server/dao/sql/user/UserRepository.java | 7 +- .../dao/sql/widget/JpaWidgetTypeDao.java | 6 +- .../dao/sql/widget/JpaWidgetsBundleDao.java | 7 +- .../dao/sql/widget/WidgetTypeRepository.java | 6 +- .../sql/widget/WidgetsBundleRepository.java | 6 +- edqs/pom.xml | 5 +- .../edqs/DummyQueueRoutingInfoService.java | 2 +- .../edqs/DummyTenantRoutingInfoService.java | 2 +- .../edqs/ThingsboardEdqsApplication.java | 4 +- edqs/src/main/resources/edqs.yml | 2 +- edqs/src/main/resources/logback.xml | 2 +- .../server/edqs/repo/AbstractEDQTest.java | 51 +--- .../edqs/repo/ApiUsageStateFilterTest.java | 5 +- .../edqs/repo/AssetSearchQueryFilterTest.java | 114 ++------ .../server/edqs/repo/AssetTypeFilterTest.java | 41 ++- .../repo/DeviceSearchQueryFilterTest.java | 133 ++-------- .../edqs/repo/DeviceTypeFilterTest.java | 15 +- .../edqs/repo/EdgeSearchQueryFilterTest.java | 81 ++---- .../server/edqs/repo/EdgeTypeFilterTest.java | 25 +- .../repo/EntitiesByGroupIdFilterTest.java | 161 ------------ .../repo/EntitiesByGroupNameFilterTest.java | 140 ---------- .../edqs/repo/EntityGroupListFilterTest.java | 189 -------------- .../edqs/repo/EntityGroupNameFilterTest.java | 186 ------------- .../edqs/repo/EntityListFilterTest.java | 71 +---- .../edqs/repo/EntityNameFilterTest.java | 15 +- .../edqs/repo/EntityTypeFilterTest.java | 15 +- .../repo/EntityViewSearchQueryFilterTest.java | 131 ++-------- .../edqs/repo/EntityViewTypeFilterTest.java | 25 +- .../edqs/repo/RelationsQueryFilterTest.java | 91 +------ .../server/edqs/repo/RepositoryUtilsTest.java | 2 +- .../edqs/repo/SchedulerEventFilterTest.java | 129 --------- .../edqs/repo/SingleEntityFilterTest.java | 55 +--- .../edqs/repo/StateEntityOwnerFilterTest.java | 81 ------ pom.xml | 1 + 163 files changed, 557 insertions(+), 3488 deletions(-) delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index bf8e096059..04633892e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.edqs.Entity; import org.thingsboard.server.common.data.edqs.LatestTsKv; import org.thingsboard.server.common.data.edqs.fields.EntityFields; -import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -37,7 +36,6 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.entity.EntityDaoRegistry; -import org.thingsboard.server.dao.group.EntityGroupDao; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -46,39 +44,16 @@ import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; import org.thingsboard.server.dao.tenant.TenantDao; -import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.common.data.ObjectType.API_USAGE_STATE; -import static org.thingsboard.server.common.data.ObjectType.ASSET; -import static org.thingsboard.server.common.data.ObjectType.ASSET_PROFILE; import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; -import static org.thingsboard.server.common.data.ObjectType.BLOB_ENTITY; -import static org.thingsboard.server.common.data.ObjectType.CONVERTER; -import static org.thingsboard.server.common.data.ObjectType.CUSTOMER; -import static org.thingsboard.server.common.data.ObjectType.DASHBOARD; -import static org.thingsboard.server.common.data.ObjectType.DEVICE; -import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE; -import static org.thingsboard.server.common.data.ObjectType.EDGE; -import static org.thingsboard.server.common.data.ObjectType.ENTITY_GROUP; -import static org.thingsboard.server.common.data.ObjectType.ENTITY_VIEW; -import static org.thingsboard.server.common.data.ObjectType.INTEGRATION; import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; -import static org.thingsboard.server.common.data.ObjectType.QUEUE_STATS; import static org.thingsboard.server.common.data.ObjectType.RELATION; -import static org.thingsboard.server.common.data.ObjectType.ROLE; -import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN; -import static org.thingsboard.server.common.data.ObjectType.SCHEDULER_EVENT; -import static org.thingsboard.server.common.data.ObjectType.TENANT; -import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE; -import static org.thingsboard.server.common.data.ObjectType.USER; -import static org.thingsboard.server.common.data.ObjectType.WIDGETS_BUNDLE; -import static org.thingsboard.server.common.data.ObjectType.WIDGET_TYPE; +import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; @Slf4j public abstract class EdqsSyncService { @@ -94,8 +69,6 @@ public abstract class EdqsSyncService { @Autowired private RelationRepository relationRepository; @Autowired - private EntityGroupDao entityGroupDao; - @Autowired private TsKvLatestRepository tsKvLatestRepository; @Autowired @Lazy @@ -106,12 +79,6 @@ public abstract class EdqsSyncService { private final Map counters = new ConcurrentHashMap<>(); - public static final Set edqsTenantTypes = EnumSet.of( - TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, - RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, - BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS - ); - public abstract boolean isSyncNeeded(); public void sync() { @@ -119,9 +86,7 @@ public abstract class EdqsSyncService { long startTs = System.currentTimeMillis(); counters.clear(); - syncTenants(); syncTenantEntities(); - syncEntityGroups(); syncRelations(); loadKeyDictionary(); syncAttributes(); @@ -139,49 +104,28 @@ public abstract class EdqsSyncService { edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); } - private void syncTenants() { - log.info("Synchronizing tenants to EDQS"); - long ts = System.currentTimeMillis(); - var tenants = new PageDataIterable<>(tenantDao::findAllFields, 10000); - for (EntityFields entityFields : tenants) { - TenantId tenantId = TenantId.fromUUID(entityFields.getId()); - entityInfoMap.put(entityFields.getId(), new EntityIdInfo(EntityType.TENANT, tenantId)); - process(tenantId, TENANT, new Entity(EntityType.TENANT, entityFields)); - } - process(TenantId.SYS_TENANT_ID, TENANT, new Entity(EntityType.TENANT, new TenantFields(TenantId.SYS_TENANT_ID.getId(), Long.MAX_VALUE))); - log.info("Finished synchronizing tenants to EDQS in {} ms", (System.currentTimeMillis() - ts)); - } - private void syncTenantEntities() { for (ObjectType type : edqsTenantTypes) { - log.info("Synchronizing tenant {} entities to EDQS", type); + log.info("Synchronizing {} entities to EDQS", type); long ts = System.currentTimeMillis(); EntityType entityType = type.toEntityType(); Dao dao = entityDaoRegistry.getDao(entityType); - var entities = new PageDataIterable<>(dao::findAllFields, 10000); - for (EntityFields entityFields : entities) { - TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); - entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); - process(tenantId, type, new Entity(type.toEntityType(), entityFields)); - } - log.info("Finished synchronizing tenant {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); - } - } - - private void syncEntityGroups() { - log.info("Synchronizing entity groups to EDQS"); - long ts = System.currentTimeMillis(); - var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 30000); - for (EntityFields groupFields : entityGroups) { - EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId()); - if (entityIdInfo != null) { - entityInfoMap.put(groupFields.getId(), new EntityIdInfo(EntityType.ENTITY_GROUP, entityIdInfo.tenantId())); - process(entityIdInfo.tenantId(), ENTITY_GROUP, new Entity(EntityType.ENTITY_GROUP, groupFields)); - } else { - log.info("Entity group owner not found: " + groupFields.getOwnerId()); + UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + while (true) { + var batch = dao.findNextBatch(lastFromEntityId, 10000); + if (batch.isEmpty()) { + break; + } + for (EntityFields entityFields : batch) { + TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); + process(tenantId, type, new Entity(entityType, entityFields)); + } + EntityFields lastRecord = batch.get(batch.size() - 1); + lastFromEntityId = lastRecord.getId(); } + log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); } - log.info("Finished synchronizing entity groups to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private void syncRelations() { @@ -215,7 +159,7 @@ public abstract class EdqsSyncService { private void processRelationBatch(List relations) { for (RelationEntity relation : relations) { - if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup()) || (RelationTypeGroup.FROM_ENTITY_GROUP.name().equals(relation.getRelationTypeGroup()))) { + if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) { EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); if (entityIdInfo != null) { process(entityIdInfo.tenantId(), RELATION, relation.toData()); @@ -284,7 +228,7 @@ public abstract class EdqsSyncService { int lastKey = Integer.MIN_VALUE; while (true) { - List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); if (batch.isEmpty()) { break; } @@ -332,6 +276,7 @@ public abstract class EdqsSyncService { return strKey; } - public record EntityIdInfo(EntityType entityType, TenantId tenantId) {} + public record EntityIdInfo(EntityType entityType, TenantId tenantId) { + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index daa1c92c1c..a75461faaa 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -28,13 +28,14 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; import java.util.concurrent.TimeUnit; +import java.util.function.BiPredicate; import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { - "queue.type=kafka", // uncomment to use Kafka - "queue.kafka.bootstrap.servers=10.7.1.254:9092", +// "queue.type=kafka", // uncomment to use Kafka +// "queue.kafka.bootstrap.servers=10.7.1.254:9092", "queue.edqs.sync_enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" @@ -64,4 +65,9 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { result -> result == expectedResult); } + @Override + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> condition.test(result, expectedResult)); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index ad554f11ba..3941f16b88 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -73,6 +73,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.BiPredicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @@ -80,7 +81,7 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest -public class EntityQueryControllerTest extends AbstractControllerTest { +public abstract class EntityQueryControllerTest extends AbstractControllerTest { private static final String CUSTOMER_USER_EMAIL = "entityQueryCustomer@thingsboard.org"; private static final String TENANT_PASSWORD = "testPassword1"; @@ -157,17 +158,10 @@ public class EntityQueryControllerTest extends AbstractControllerTest { @Test public void testSysAdminCountEntitiesByQuery() throws Exception { - loginSysAdmin(); - - EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); - allDeviceFilter.setEntityType(EntityType.DEVICE); - EntityCountQuery query = new EntityCountQuery(allDeviceFilter); - Long initialCount = countByQuery(query); - loginTenantAdmin(); List devices = new ArrayList<>(); - String devicePrefix = "Device" + RandomStringUtils.random(5); + String devicePrefix = "Device" + RandomStringUtils.randomAlphabetic(5); for (int i = 0; i < 97; i++) { Device device = new Device(); device.setName(devicePrefix + i); @@ -177,18 +171,18 @@ public class EntityQueryControllerTest extends AbstractControllerTest { Thread.sleep(1); } DeviceTypeFilter filter = new DeviceTypeFilter(); - filter.setDeviceType("default"); + filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter(""); loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); - countByQueryAndCheck(countQuery, initialCount + 97); + countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); - filter.setDeviceType("unknown"); + filter.setDeviceTypes(List.of("unknown")); countByQueryAndCheck(countQuery, 0); - filter.setDeviceType("default"); + filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter(devicePrefix + "1"); countByQueryAndCheck(countQuery, 11); @@ -199,7 +193,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { countQuery = new EntityCountQuery(entityListFilter); countByQueryAndCheck(countQuery, 97); - countByQueryAndCheck(query, initialCount + 97); + countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); } @Test @@ -847,9 +841,9 @@ public class EntityQueryControllerTest extends AbstractControllerTest { var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == expectedNumOfDevices; }); - if (expectedNumOfDevices == 0) { - return; - } + if (expectedNumOfDevices == 0) { + return; + } var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); @@ -870,7 +864,8 @@ public class EntityQueryControllerTest extends AbstractControllerTest { } protected PageData findByQuery(EntityDataQuery query) throws Exception { - return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); + return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() { + }); } protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception { @@ -911,4 +906,9 @@ public class EntityQueryControllerTest extends AbstractControllerTest { return numericFilter; } + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) throws Exception { + Long result = countByQuery(query); + assertThat(condition.test(result, expectedResult)).isTrue(); + return result; + } } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index eea7e0eb7f..cf24deb7e4 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -21,7 +21,6 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.permission.MergedUserPermissions; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -53,19 +52,19 @@ public class EdqsEntityServiceTest extends EntityServiceTest { } @Override - protected PageData findByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityDataQuery query, long expectedResultSize) { - return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, permissions, query), + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), result -> result.getTotalElements() == expectedResultSize); } @Override protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { - return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), mergedUserPermissionsPE, countQuery, expectedResult); + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); } @Override - protected long countByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityCountQuery query, int expectedResult) { - return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, permissions, query), + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query), result -> result == expectedResult); } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 04bad95be7..fab4cba3f5 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -489,7 +489,7 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - countByQueryAndCheck(countQuery, 9); + countByQueryAndCheck(countQuery, 3); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); @@ -1444,30 +1444,16 @@ public class EntityServiceTest extends AbstractControllerTest { assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); // find by customer user with generic permission - MergedUserPermissions mergedGenericPermission = new MergedUserPermissions(Map.of(Resource.DEVICE, Set.of(Operation.READ)), Collections.emptyMap()); - PageData customerResults = findByQueryAndCheck(customerId, mergedGenericPermission, query, 1); + PageData customerResults = findByQueryAndCheck(customerId, query, 1); String cutomerDeviceName = customerResults.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(cutomerDeviceName).isEqualTo(customerDevices.get(0).getName()); - // find by customer user with group permission - MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(customerDeviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); - PageData result2 = findByQueryAndCheck(customerId, mergedGroupOnlyPermission, query, 1); - - String resultDeviceName2 = result2.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(resultDeviceName2).isEqualTo(customerDevices.get(0).getName()); - // try to find tenant device by customer user SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); - findByQueryAndCheck(customerId, mergedGenericPermission, customerQuery2, 0); - - // find by tenant user with group permission - PageData results3 = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), mergedGroupOnlyPermission, query, 1); - - String deviceName3 = results3.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(deviceName3).isEqualTo(customerDevices.get(0).getName()); + findByQueryAndCheck(customerId, customerQuery2, 0); } private List getResultDeviceIds(PageData result) { @@ -1634,230 +1620,6 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } - @Test - public void testFindEntityDataByRelationQuery_blobEntity_customerLevel() { - final int deviceCnt = 2; - final int relationsCnt = 3; - final int blobEntitiesCnt = deviceCnt * relationsCnt; - - Customer customer = new Customer(); - customer.setTenantId(tenantId); - customer.setTitle("Customer Relation Query"); - customer = customerService.saveCustomer(customer); - - List devices = new ArrayList<>(); - for (int i = 0; i < deviceCnt; i++) { - Device device = new Device(); - device.setTenantId(tenantId); - device.setName("Device relation query " + i); - device.setCustomerId(customer.getId()); - device.setType("default"); - devices.add(deviceService.saveDevice(device)); - } - - List blobEntities = new ArrayList<>(); - for (int i = 0; i < blobEntitiesCnt; i++) { - BlobEntity blobEntity = new BlobEntity(); - blobEntity.setName("Blob relation query " + i); - blobEntity.setTenantId(tenantId); - blobEntity.setContentType("image/png"); - blobEntity.setData(ByteBuffer.allocate(1024)); - blobEntity.setCustomerId(customer.getId()); - blobEntity.setType("Report"); - blobEntities.add(blobEntityService.saveBlobEntity(blobEntity)); - } - - for (int i = 0; i < deviceCnt; i++) { - for (int j = 0; j < relationsCnt; j++) { - EntityRelation relationEntity = new EntityRelation(); - relationEntity.setFrom(devices.get(i).getId()); - relationEntity.setTo(blobEntities.get(j + (i * relationsCnt)).getId()); - relationEntity.setTypeGroup(RelationTypeGroup.COMMON); - relationEntity.setType("fileAttached"); - relationService.saveRelation(tenantId, relationEntity); - } - } - - MergedUserPermissions mergedUserPermissions = new MergedUserPermissions(Map.of(ALL, Set.of(Operation.ALL)), Collections.emptyMap()); - - RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("fileAttached", Collections.singletonList(EntityType.BLOB_ENTITY)); - RelationsQueryFilter filter = new RelationsQueryFilter(); - filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); - filter.setDirection(EntitySearchDirection.FROM); - EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); - - for (Device device : devices) { - filter.setRootEntity(device.getId()); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); - findByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); - countByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); - /* - In order to be careful with updating Relation Query while adding new Entity Type, - this checkup will help to find place, where you could check the correctness of building query - */ - Assert.assertEquals(38, EntityType.values().length); - } - } - - @Test - public void testFindEntitiesByRelationEntityTypeFilterWithTenantGroupPermission() { - final int assetCount = 2; - final int relationsCnt = 4; - final int deviceEntitiesCnt = assetCount * relationsCnt; - - EntityGroup deviceGroup = new EntityGroup(); - deviceGroup.setName("Device Tenant Level Group"); - deviceGroup.setOwnerId(tenantId); - deviceGroup.setTenantId(tenantId); - deviceGroup.setType(EntityType.DEVICE); - deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); - - List assets = new ArrayList<>(); - for (int i = 0; i < assetCount; i++) { - Asset building = new Asset(); - building.setTenantId(tenantId); - building.setName("Building _" + i); - building.setType("building"); - building = assetService.saveAsset(building); - assets.add(building); - } - - List devices = new ArrayList<>(); - for (int i = 0; i < deviceEntitiesCnt; i++) { - Device device = new Device(); - device.setTenantId(tenantId); - device.setName("Test device " + i); - device.setType("default"); - Device savedDevice = deviceService.saveDevice(device); - devices.add(savedDevice); - if (i % 2 == 0) { - entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); - } - } - - for (int i = 0; i < assetCount; i++) { - for (int j = 0; j < relationsCnt; j++) { - EntityRelation relationEntity = new EntityRelation(); - relationEntity.setFrom(assets.get(i).getId()); - relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); - relationEntity.setTypeGroup(RelationTypeGroup.COMMON); - relationEntity.setType("contains"); - relationService.saveRelation(tenantId, relationEntity); - } - } - - MergedUserPermissions groupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), - Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); - - RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); - RelationsQueryFilter filter = new RelationsQueryFilter(); - filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); - filter.setDirection(EntitySearchDirection.FROM); - EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); - List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); - - for (Asset asset : assets) { - filter.setRootEntity(asset.getId()); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); - countByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); - } - } - - @Test - public void testFindEntitiesWithRelationEntityTypeFilterByCustomerUser() { - Customer customer = new Customer(); - customer.setTenantId(tenantId); - customer.setTitle("Customer Relation Query"); - customer = customerService.saveCustomer(customer); - - final int assetCount = 2; - final int relationsCnt = 4; - final int deviceEntitiesCnt = assetCount * relationsCnt; - - EntityGroup deviceGroup = new EntityGroup(); - deviceGroup.setName("Device Tenant Level Group"); - deviceGroup.setOwnerId(customer.getId()); - deviceGroup.setTenantId(tenantId); - deviceGroup.setType(EntityType.DEVICE); - deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); - - List assets = new ArrayList<>(); - for (int i = 0; i < assetCount; i++) { - Asset building = new Asset(); - building.setTenantId(tenantId); - building.setCustomerId(customer.getId()); - building.setName("Building _" + i); - building.setType("building"); - building = assetService.saveAsset(building); - assets.add(building); - } - - List devices = new ArrayList<>(); - for (int i = 0; i < deviceEntitiesCnt; i++) { - Device device = new Device(); - device.setTenantId(tenantId); - device.setCustomerId(customer.getId()); - device.setName("Test device " + i); - device.setType("default"); - Device savedDevice = deviceService.saveDevice(device); - devices.add(savedDevice); - if (i % 2 == 0) { - entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); - } - } - - for (int i = 0; i < assetCount; i++) { - for (int j = 0; j < relationsCnt; j++) { - EntityRelation relationEntity = new EntityRelation(); - relationEntity.setFrom(assets.get(i).getId()); - relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); - relationEntity.setTypeGroup(RelationTypeGroup.COMMON); - relationEntity.setType("contains"); - relationService.saveRelation(tenantId, relationEntity); - } - } - - MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - MergedUserPermissions mergedGenericOnlyPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.emptyMap()); - MergedUserPermissions mergedGenericAndGroupPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); - RelationsQueryFilter filter = new RelationsQueryFilter(); - filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); - filter.setDirection(EntitySearchDirection.FROM); - EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); - List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - - for (Asset asset : assets) { - filter.setRootEntity(asset.getId()); - - //check by user with generic permission - PageData relationsResult = findByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); - countByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); - - //check by user with generic and group permission - PageData relationsResult1 = findByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); - countByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); - - //check by other customer user with group only permission - PageData relationsResult2 = findByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); - long relationsResultCnt2 = countByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); - - Assert.assertEquals(relationsCnt / 2, relationsResult2.getData().size()); - Assert.assertEquals(relationsCnt / 2, relationsResultCnt2); - - //check by other customer user with generic and group only permission - PageData relationsResult3 = findByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); - long relationsResultCnt3 = countByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); - - Assert.assertEquals(relationsCnt / 2, relationsResult3.getData().size()); - Assert.assertEquals(relationsCnt / 2, relationsResultCnt3); - } - } @Test public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index c78998cd6a..d3aa547bf0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -30,16 +30,8 @@ public enum ObjectType { RULE_CHAIN, OTA_PACKAGE, RESOURCE, - ROLE, - ENTITY_GROUP, - DEVICE_GROUP_OTA_PACKAGE, - GROUP_PERMISSION, - BLOB_ENTITY, - SCHEDULER_EVENT, EVENT, RULE_NODE, - CONVERTER, - INTEGRATION, USER, USER_CREDENTIALS, USER_AUTH_SETTINGS, @@ -62,8 +54,6 @@ public enum ObjectType { NOTIFICATION_TARGET, NOTIFICATION_TEMPLATE, NOTIFICATION_RULE, - WHITE_LABELING, - CUSTOM_TRANSLATION, ALARM_COMMENT, ALARM_TYPE, API_USAGE_STATE, @@ -75,16 +65,15 @@ public enum ObjectType { LATEST_TS_KV; public static final Set edqsTenantTypes = EnumSet.of( - TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, - RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, - BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS + TENANT, TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS ); public static final Set edqsTypes = new HashSet<>(edqsTenantTypes); public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); static { - edqsTypes.addAll(Arrays.asList(TENANT, ENTITY_GROUP, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + edqsTypes.addAll(Arrays.asList(TENANT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); } public EntityType toEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java index af1640b7be..fa57f9d0f4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java @@ -15,10 +15,15 @@ */ package org.thingsboard.server.common.data.edqs.fields; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; @@ -27,7 +32,30 @@ import java.util.UUID; @SuperBuilder public class DashboardFields extends AbstractEntityFields { - public DashboardFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { - super(id, createdTime, tenantId, customerId, name, version); + private static ObjectMapper objectMapper = new ObjectMapper(); + private List assignedCustomerIds; + + public DashboardFields(UUID id, long createdTime, UUID tenantId, String assignedCustomers, String name, Long version) { + super(id, createdTime, tenantId, name, version); + this.assignedCustomerIds = getCustomerIds(assignedCustomers); + } + + private static List getCustomerIds(String assignedCustomers) { + List ids = new ArrayList<>(); + if (assignedCustomers == null || assignedCustomers.isEmpty()) { + return ids; + } + try { + JsonNode rootNode = objectMapper.readTree(assignedCustomers); + for (JsonNode node : rootNode) { + String idStr = node.path("customerId").path("id").asText(); + if (!idStr.isEmpty()) { + ids.add(UUID.fromString(idStr)); + } + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return ids; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java index 7536586f57..ba9f3b0e9b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -19,6 +19,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.thingsboard.server.common.data.id.EntityId; +import java.util.Collections; +import java.util.List; import java.util.UUID; public interface EntityFields { @@ -37,6 +39,10 @@ public interface EntityFields { return null; } + default List getAssignedCustomerIds() { + return Collections.emptyList(); + } + default long getCreatedTime() { return 0; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java index 5635566cc4..01c32f820e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java @@ -19,6 +19,8 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; +import java.util.UUID; + @Data @NoArgsConstructor @SuperBuilder @@ -27,4 +29,9 @@ public class EntityViewFields extends AbstractEntityFields { private String type; private String additionalInfo; + public EntityViewFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, String type, String additionalInfo, Long version) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.additionalInfo = additionalInfo; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index f1cf51bf8b..6543a6d779 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -85,7 +85,6 @@ public class FieldsUtil { return CustomerFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) - .customerId(getCustomerId(entity.getCustomerId())) .name(entity.getTitle()) .additionalInfo(getText(entity.getAdditionalInfo())) .email(entity.getEmail()) @@ -199,7 +198,6 @@ public class FieldsUtil { return DashboardFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) - .customerId(getCustomerId(entity.getCustomerId())) .name(entity.getTitle()) .version(entity.getVersion()) .build(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java index 135061a842..b776d18551 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.common.data.edqs.query; -import com.fasterxml.jackson.annotation.JsonIncludeProperties; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.permission.MergedUserPermissions; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -32,7 +30,5 @@ public class EdqsRequest { private EntityDataQuery entityDataQuery; private EntityCountQuery entityCountQuery; - @JsonIncludeProperties({"genericPermissions", "groupPermissions"}) - private MergedUserPermissions userPermissions; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java index 0c45812690..70e25ba14e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java @@ -30,12 +30,10 @@ import java.util.Map; public class QueryResult { private final EntityId entityId; - private final boolean readAttrs; - private final boolean readTs; private final Map> latest; public EntityData toOldEntityData() { - return new EntityData(entityId, readAttrs, readTs, latest, Collections.emptyMap(), Collections.emptyMap()); + return new EntityData(entityId, latest, Collections.emptyMap(), Collections.emptyMap()); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java index b377d63ee4..b8b3f51aec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java @@ -44,4 +44,7 @@ public class QueryContext { this(tenantId, customerId, entityType, false); } + public boolean isTenantUser() { + return customerId == null || customerId.isNullUid(); + } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java index 9c3536dea1..6c648daf02 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java @@ -18,15 +18,13 @@ package org.thingsboard.server.common.data.queue; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasEntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; @EqualsAndHashCode(callSuper = true) @Data -public class QueueStats extends BaseData implements HasTenantId, HasEntityType { +public class QueueStats extends BaseData implements HasTenantId { private TenantId tenantId; private String queueName; private String serviceId; @@ -38,8 +36,4 @@ public class QueueStats extends BaseData implements HasTenantId, H super(id); } - @Override - public EntityType getEntityType() { - return EntityType.QUEUE_STATS; - } } diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml index 40e89acee6..e58c7c97a0 100644 --- a/common/edqs/pom.xml +++ b/common/edqs/pom.xml @@ -1,6 +1,6 @@ A2, A1 --Contains--> D1. A1 --Manages--> D2. - createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); - - MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), - new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); - - PageData relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(1, relationsResult.getData().size()); - Assert.assertTrue(checkContains(relationsResult, da1)); - - relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(0, relationsResult.getData().size()); - } - @Test public void testFindCustomerDevices() { UUID ta1 = createAsset("T A1"); @@ -169,55 +136,15 @@ public class RelationsQueryFilterTest extends AbstractEDQTest { Assert.assertEquals(0, relationsResult.getData().size()); } - @Test - public void testFindCustomerDevicesGroupsOnly() { - UUID ta1 = createAsset("T A1"); - UUID ta2 = createAsset("T A2"); - UUID da1 = createDevice(customerId, "T D1"); - UUID da2 = createDevice(customerId, "T D2"); - UUID da3 = createDevice(customerId, "T D3"); - - UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); - createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - - // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. - createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); - createRelation(EntityType.DEVICE, da2, EntityType.DEVICE, da3, "Contains"); - - MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), - new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); - - PageData relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(2, relationsResult.getData().size()); - Assert.assertTrue(checkContains(relationsResult, da1)); - Assert.assertTrue(checkContains(relationsResult, da3)); - - relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(1, relationsResult.getData().size()); - Assert.assertTrue(checkContains(relationsResult, da2)); - } - private PageData filter(EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(RepositoryUtils.ALL_READ_PERMISSIONS, rootId, relationEntityTypeFilters); - } - - private PageData filter(MergedUserPermissions permissions, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(permissions, null, rootId, relationEntityTypeFilters); + return filter(null, rootId, relationEntityTypeFilters); } private PageData filter(CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, rootId, relationEntityTypeFilters); - } - - private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(permissions, customerId, rootId, 3, false, relationEntityTypeFilters); + return filter(customerId, rootId, 3, false, relationEntityTypeFilters); } - private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { + private PageData filter(CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { RelationsQueryFilter filter = new RelationsQueryFilter(); filter.setRootEntity(rootId); filter.setFilters(Arrays.asList(relationEntityTypeFilters)); @@ -227,7 +154,7 @@ public class RelationsQueryFilterTest extends AbstractEDQTest { EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "T"); EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); } } diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index 42706a796a..48a72a9cc6 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2024 ThingsBoard, Inc. + * Copyright © 2016-2024 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java deleted file mode 100644 index 8ebbff534d..0000000000 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Copyright © 2016-2024 ThingsBoard, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.edqs.repo; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityKeyValueType; -import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.query.KeyFilter; -import org.thingsboard.server.common.data.query.SchedulerEventFilter; -import org.thingsboard.server.common.data.query.StringFilterPredicate; -import org.thingsboard.server.edqs.util.RepositoryUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.UUID; - -public class SchedulerEventFilterTest extends AbstractEDQTest { - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - @Test - public void testFindTenantSchedulerEvents() { - UUID dashboardId = createDashboard("test dashboard"); - UUID deviceId = createDevice("test device"); - - UUID eventId1 = createSchedulerEvent("Update attributes", new DeviceId(deviceId), "Turn off device"); - UUID eventId2 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate morning report"); - UUID eventId3 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate evening report"); - - // find all scheduler events with type "Generate report" - var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); - Assert.assertEquals(2, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - Assert.assertTrue(checkContains(result, eventId3)); - - // find all scheduler events for device originator - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId1)); - - // find all scheduler events with name "%morning%" - KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - } - - @Test - public void testFindCustomerEdges() { - UUID dashboardId = createDashboard( "test dashboard"); - UUID deviceId = createDevice("test device"); - - UUID eventId1 = createSchedulerEvent(customerId.getId(), "Update attributes", new DeviceId(deviceId), "Turn off device"); - UUID eventId2 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate morning report"); - UUID eventId3 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate evening report"); - - // find all scheduler events with type "Generate report" - var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); - Assert.assertEquals(2, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - Assert.assertTrue(checkContains(result, eventId3)); - - // find all scheduler events for device originator - result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId1)); - - // find all scheduler events with name "%morning%" - KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); - result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - } - - private static EntityDataQuery getSchedulerEventQuery(String eventType, EntityId entityId, List keyFilters) { - SchedulerEventFilter filter = new SchedulerEventFilter(); - filter.setEventType(eventType); - filter.setOriginator(entityId); - var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); - - var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); - var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); - - return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - } - - private static KeyFilter getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { - KeyFilter nameFilter = new KeyFilter(); - nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); - var predicate = new StringFilterPredicate(); - predicate.setIgnoreCase(ignoreCase); - predicate.setOperation(operation); - predicate.setValue(new FilterPredicateValue<>(predicateValue)); - nameFilter.setPredicate(predicate); - nameFilter.setValueType(EntityKeyValueType.STRING); - return nameFilter; - } - -} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java index 8755191e39..c7de17e33c 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2024 ThingsBoard, Inc. + * Copyright © 2016-2024 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,16 +22,10 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edqs.LatestTsKv; -import org.thingsboard.server.common.data.edqs.query.QueryResult; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EntityGroupId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; -import org.thingsboard.server.common.data.permission.MergedUserPermissions; -import org.thingsboard.server.common.data.permission.Operation; -import org.thingsboard.server.common.data.permission.Resource; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; @@ -42,12 +36,8 @@ import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.edqs.util.RepositoryUtils; import java.util.Arrays; -import java.util.Map; -import java.util.Set; import java.util.UUID; public class SingleEntityFilterTest extends AbstractEDQTest { @@ -72,7 +62,7 @@ public class SingleEntityFilterTest extends AbstractEDQTest { addOrUpdate(EntityType.DEVICE, device); addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); - var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + var result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); Assert.assertEquals(1, result.getTotalElements()); var first = result.getData().get(0); @@ -80,13 +70,13 @@ public class SingleEntityFilterTest extends AbstractEDQTest { Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); Assert.assertEquals(0, result.getTotalElements()); device.setCustomerId(customerId); addOrUpdate(EntityType.DEVICE, device); - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); Assert.assertEquals(1, result.getTotalElements()); first = result.getData().get(0); Assert.assertEquals(deviceId, first.getEntityId()); @@ -94,39 +84,6 @@ public class SingleEntityFilterTest extends AbstractEDQTest { Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); } - @Test - public void testFindTenantDeviceWithGenericAndGroupPermission() { - UUID deviceId = createDevice(customerId, "LoRa-customer-1"); - UUID deviceId2 = createDevice(customerId, "LoRa-customer-2"); - UUID deviceId3 = createDevice(customerId, "LoRa-customer-3"); - - // add device and device 2 to Group A - UUID groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); - createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - - // add device and device 2 to Group A - UUID groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); - createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - - MergedUserPermissions genericAndGroupAPermission = new MergedUserPermissions( - Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - var result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupAPermission, getEntityDataQuery(new DeviceId(deviceId2)), false); - Assert.assertEquals(1, result.getTotalElements()); - QueryResult queryResult = result.getData().get(0); - Assert.assertEquals(deviceId2, queryResult.getEntityId().getId()); - Assert.assertEquals("LoRa-customer-2", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); - - // find device without permission - MergedUserPermissions genericAndGroupBPermission = new MergedUserPermissions( - Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupBPermission, getEntityDataQuery(new DeviceId(deviceId3)), false); - Assert.assertEquals(1, result.getTotalElements()); - queryResult = result.getData().get(0); - Assert.assertEquals(deviceId3, queryResult.getEntityId().getId()); - Assert.assertEquals("LoRa-customer-3", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); - } - @Test public void testFindCustomerDevice() { DeviceId deviceId = new DeviceId(UUID.randomUUID()); @@ -139,13 +96,13 @@ public class SingleEntityFilterTest extends AbstractEDQTest { addOrUpdate(EntityType.DEVICE, device); addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); - var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); Assert.assertEquals(0, result.getTotalElements()); device.setCustomerId(customerId); addOrUpdate(EntityType.DEVICE, device); - result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); Assert.assertEquals(1, result.getTotalElements()); var first = result.getData().get(0); diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java deleted file mode 100644 index f0bb4c4e0b..0000000000 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright © 2016-2024 ThingsBoard, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.edqs.repo; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityKeyValueType; -import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.query.KeyFilter; -import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; -import org.thingsboard.server.common.data.query.StringFilterPredicate; -import org.thingsboard.server.edqs.util.RepositoryUtils; - -import java.util.Arrays; -import java.util.UUID; - -public class StateEntityOwnerFilterTest extends AbstractEDQTest { - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - @Test - public void testFindCustomerDeviceOwner() { - UUID customerId = UUID.randomUUID(); - createCustomer(customerId, null, "Customer A"); - UUID deviceId = createDevice(new CustomerId(customerId), "LoRa-1"); - - var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(deviceId)), false); - - Assert.assertEquals(1, result.getTotalElements()); - var customer = result.getData().get(0); - Assert.assertEquals(customerId, customer.getEntityId().getId()); - Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); - } - - private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { - StateEntityOwnerFilter filter = new StateEntityOwnerFilter(); - filter.setSingleEntity(deviceId); - var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); - - var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); - KeyFilter nameFilter = new KeyFilter(); - nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); - var predicate = new StringFilterPredicate(); - predicate.setIgnoreCase(false); - predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); - predicate.setValue(new FilterPredicateValue<>("LoRa-")); - nameFilter.setPredicate(predicate); - nameFilter.setValueType(EntityKeyValueType.STRING); - - return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); - } - -} diff --git a/pom.xml b/pom.xml index 42747bae46..c809651aa2 100755 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ 1.7.0 4.4.0 2.2.14 + 0.6.12 3.12.1 2.0.0-M15 2.10.1 From 60736e12377d2f6411e873c2e9f7f827f71b8c42 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 4 Feb 2025 12:23:35 +0200 Subject: [PATCH 04/51] fixed entity removal from CustomerData --- .../server/service/edqs/EdqsSyncService.java | 6 ++-- .../EdqsEntityQueryControllerTest.java | 5 --- .../controller/EntityQueryControllerTest.java | 18 +++++----- .../server/common/data/ObjectType.java | 2 +- .../processor/AbstractQueryProcessor.java | 2 +- .../AbstractRelationQueryProcessor.java | 5 ++- .../processor/EntityListQueryProcessor.java | 2 +- .../processor/SingleEntityQueryProcessor.java | 2 +- .../server/edqs/repo/TenantRepo.java | 35 ++++++++++--------- .../dao/queue/BaseQueueStatsService.java | 2 +- .../server/dao/queue/QueueStatsDao.java | 7 ++-- .../dao/sql/queue/JpaQueueStatsDao.java | 8 ++++- 12 files changed, 48 insertions(+), 46 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 04633892e5..c2750ae840 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -110,9 +110,9 @@ public abstract class EdqsSyncService { long ts = System.currentTimeMillis(); EntityType entityType = type.toEntityType(); Dao dao = entityDaoRegistry.getDao(entityType); - UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); while (true) { - var batch = dao.findNextBatch(lastFromEntityId, 10000); + var batch = dao.findNextBatch(lastId, 10000); if (batch.isEmpty()) { break; } @@ -122,7 +122,7 @@ public abstract class EdqsSyncService { process(tenantId, type, new Entity(entityType, entityFields)); } EntityFields lastRecord = batch.get(batch.size() - 1); - lastFromEntityId = lastRecord.getId(); + lastId = lastRecord.getId(); } log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index a75461faaa..baf9f98993 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -65,9 +65,4 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { result -> result == expectedResult); } - @Override - protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) { - return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), - result -> condition.test(result, expectedResult)); - } } diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 3941f16b88..508f66e584 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -81,7 +81,7 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest -public abstract class EntityQueryControllerTest extends AbstractControllerTest { +public class EntityQueryControllerTest extends AbstractControllerTest { private static final String CUSTOMER_USER_EMAIL = "entityQueryCustomer@thingsboard.org"; private static final String TENANT_PASSWORD = "testPassword1"; @@ -158,6 +158,13 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { @Test public void testSysAdminCountEntitiesByQuery() throws Exception { + loginSysAdmin(); + + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery query = new EntityCountQuery(allDeviceFilter); + countByQueryAndCheck(query, 0); + loginTenantAdmin(); List devices = new ArrayList<>(); @@ -177,7 +184,7 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); - countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); countByQueryAndCheck(countQuery, 0); @@ -193,7 +200,7 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { countQuery = new EntityCountQuery(entityListFilter); countByQueryAndCheck(countQuery, 97); - countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); + countByQueryAndCheck(countQuery, 97); } @Test @@ -906,9 +913,4 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { return numericFilter; } - protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) throws Exception { - Long result = countByQuery(query); - assertThat(condition.test(result, expectedResult)).isTrue(); - return result; - } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index d3aa547bf0..bc5ec58213 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -73,7 +73,7 @@ public enum ObjectType { API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); static { - edqsTypes.addAll(Arrays.asList(TENANT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + edqsTypes.addAll(Arrays.asList(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); } public EntityType toEntityType() { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java index 29626dad1f..b795af08db 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java @@ -62,7 +62,7 @@ public abstract class AbstractQueryProcessor implements } } - protected static boolean checkCustomer(UUID customerId, EntityData ed) { + protected static boolean checkCustomerId(UUID customerId, EntityData ed) { return customerId.equals(ed.getCustomerId()) || (ed.getEntityType() == EntityType.DASHBOARD && ed.getFields().getAssignedCustomerIds().contains(customerId)); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java index b5eaf26e6e..2a4e72ff0b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.edqs.query.processor; import lombok.RequiredArgsConstructor; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; @@ -79,7 +78,7 @@ public abstract class AbstractRelationQueryProcessor ext } else { var customerId = ctx.getCustomerId().getId(); for (EntityData ed : entities) { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { result++; } } @@ -97,7 +96,7 @@ public abstract class AbstractRelationQueryProcessor ext var customerId = ctx.getCustomerId().getId(); List result = new ArrayList<>(); for (EntityData ed : entities) { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { result.add(toSortData(ed)); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java index 36016009bd..b319a4e13a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java @@ -41,7 +41,7 @@ public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProce @Override protected void processCustomerQuery(UUID customerId, Consumer> processor) { processAll(ed -> { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { processor.accept(ed); } }); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java index 19b32b5a93..55464b4529 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java @@ -39,7 +39,7 @@ public class SingleEntityQueryProcessor extends AbstractSingleEntityTypeQueryPro @Override protected void processCustomerQuery(UUID customerId, Consumer> processor) { processAll(ed -> { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { processor.accept(ed); } }); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 7773c760f0..68e1c42589 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -195,24 +195,20 @@ public class TenantRepo { processFields(fields); entityData.setFields(entity.getFields()); - switch (entity.getType()) { - default -> { - UUID newCustomerId = fields.getCustomerId(); - UUID oldCustomerId = entityData.getCustomerId(); - entityData.setCustomerId(newCustomerId); - if (entityIdMismatch(oldCustomerId, newCustomerId)) { - if (oldCustomerId != null) { - CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); - if (old != null) { - old.remove(entityData); - } - } - if (newCustomerId != null) { - CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); - newData.addOrUpdate(entityData); - } + UUID newCustomerId = fields.getCustomerId(); + UUID oldCustomerId = entityData.getCustomerId(); + entityData.setCustomerId(newCustomerId); + if (entityIdMismatch(oldCustomerId, newCustomerId)) { + if (oldCustomerId != null) { + CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); + if (old != null) { + old.remove(entityData); } } + if (newCustomerId != null) { + CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); + newData.addOrUpdate(entityData); + } } } finally { entityUpdateLock.unlock(); @@ -228,6 +224,13 @@ public class TenantRepo { if (removed != null) { getEntitySet(entityType).remove(removed); edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + UUID customerId = removed.getCustomerId(); + if (customerId != null) { + CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + customerData.remove(removed); + } + } } } finally { entityUpdateLock.unlock(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index efc9dab41b..e3e6edad94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -85,7 +85,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public PageData findByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findByTenantId, tenantId: [{}]", tenantId); Validator.validatePageLink(pageLink); - return queueStatsDao.findByTenantId(tenantId, pageLink); + return queueStatsDao.findAllByTenantId(tenantId, pageLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java index 5466a8afcc..018eb04d67 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java @@ -17,19 +17,16 @@ package org.thingsboard.server.dao.queue; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface QueueStatsDao extends Dao { +public interface QueueStatsDao extends Dao, TenantEntityDao { QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String queueName, String serviceId); - PageData findByTenantId(TenantId tenantId, PageLink pageLink); - void deleteByTenantId(TenantId tenantId); List findByIds(TenantId tenantId, List queueStatsIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index 97fedd693a..41e2de02a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -21,6 +21,7 @@ import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; @@ -62,10 +63,15 @@ public class JpaQueueStatsDao extends JpaAbstractDao findByTenantId(TenantId tenantId, PageLink pageLink) { + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { return DaoUtil.toPageData(queueStatsRepository.findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } + @Override + public ObjectType getType() { + return ObjectType.QUEUE_STATS; + } + @Override public void deleteByTenantId(TenantId tenantId) { queueStatsRepository.deleteByTenantId(tenantId.getId()); From b2bf5b9f6535bc55c233d4ba7ba13edb156bc08b Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 12:20:23 +0200 Subject: [PATCH 05/51] configurable edqs sync batch sizes --- .../server/service/edqs/DefaultEdqsService.java | 2 +- .../server/service/edqs/EdqsListener.java | 2 +- .../server/service/edqs/EdqsSyncService.java | 16 +++++++++------- .../service/edqs/KafkaEdqsSyncService.java | 2 +- .../service/edqs/LocalEdqsSyncService.java | 2 +- application/src/main/resources/thingsboard.yml | 5 ++++- .../EdqsEntityQueryControllerTest.java | 2 +- .../service/entitiy/EdqsEntityServiceTest.java | 2 +- .../test/resources/application-test.properties | 2 +- .../server/queue/edqs/EdqsComponent.java | 2 +- .../server/queue/edqs/InMemoryEdqsComponent.java | 2 +- .../server/queue/edqs/KafkaEdqsComponent.java | 2 +- .../server/dao/sql/query/DummyEdqsService.java | 2 +- 13 files changed, 24 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 160d4bb565..e7243ed3bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -81,7 +81,7 @@ import java.util.concurrent.TimeUnit; @Service @RequiredArgsConstructor @Slf4j -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") public class DefaultEdqsService implements EdqsService { private final EdqsClientQueueFactory queueFactory; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java index c4ce0c7a7e..dc8d4bf36d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @Service @RequiredArgsConstructor -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") public class EdqsListener { private final EdqsService edqsService; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index c2750ae840..80aeee8635 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.edqs; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -42,7 +43,6 @@ import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.tenant.TenantDao; import java.util.List; import java.util.Map; @@ -58,11 +58,13 @@ import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; @Slf4j public abstract class EdqsSyncService { + @Value("${queue.edqs.sync.entity_batch_size:10000}") + private int entityBatchSize; + @Value("${queue.edqs.sync.ts_batch_size:10000}") + private int tsBatchSize; @Autowired private EntityDaoRegistry entityDaoRegistry; @Autowired - private TenantDao tenantDao; - @Autowired private AttributesDao attributesDao; @Autowired private KeyDictionaryDao keyDictionaryDao; @@ -112,7 +114,7 @@ public abstract class EdqsSyncService { Dao dao = entityDaoRegistry.getDao(entityType); UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); while (true) { - var batch = dao.findNextBatch(lastId, 10000); + var batch = dao.findNextBatch(lastId, entityBatchSize); if (batch.isEmpty()) { break; } @@ -140,7 +142,7 @@ public abstract class EdqsSyncService { while (true) { List batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, - lastRelationType, lastToEntityId, lastToEntityType, 10000); + lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize); if (batch.isEmpty()) { break; } @@ -189,7 +191,7 @@ public abstract class EdqsSyncService { int lastAttributeKey = Integer.MIN_VALUE; while (true) { - List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, 10000); + List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize); if (batch.isEmpty()) { break; } @@ -228,7 +230,7 @@ public abstract class EdqsSyncService { int lastKey = Integer.MIN_VALUE; while (true) { - List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize); if (batch.isEmpty()) { break; } diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index f4e9a02b45..0c6f948770 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -27,7 +27,7 @@ import java.util.Collections; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") public class KafkaEdqsSyncService extends EdqsSyncService { private final TbKafkaSettings kafkaSettings; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java index 924bf94830..11d5894307 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java @@ -22,7 +22,7 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") public class LocalEdqsSyncService extends EdqsSyncService { private final EdqsRocksDb db; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index dd0e5948c7..e41d0c2aed 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1695,7 +1695,10 @@ queue: # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" edqs: - sync_enabled: "${TB_EDQS_SYNC_ENABLED:true}" # FIXME: disable by default before release + sync: + enabled: "${TB_EDQS_SYNC_ENABLED:true}" # Enable/disable EDQS synchronization with postgres db FIXME: disable by default before release + entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" # batch size of entities being synced with EDQS + ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" # batch size of timeseries data being synced with EDQS api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release mode: "${TB_EDQS_MODE:local}" # local or remote local: diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index baf9f98993..9bb0bbe30a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -36,7 +36,7 @@ import static org.awaitility.Awaitility.await; @TestPropertySource(properties = { // "queue.type=kafka", // uncomment to use Kafka // "queue.kafka.bootstrap.servers=10.7.1.254:9092", - "queue.edqs.sync_enabled=true", + "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" }) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index cf24deb7e4..cb377e0431 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -34,7 +34,7 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { - "queue.edqs.sync_enabled=true", + "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" }) diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index ba6863c705..8933c36db5 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -57,5 +57,5 @@ server.log_controller_error_stack_trace=true transport.gateway.dashboard.sync.enabled=false -queue.edqs.sync_enabled=false +queue.edqs.sync.enabled=false queue.edqs.api_enabled=false diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java index 838f9b4aa2..27a1e09f79 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -22,7 +22,7 @@ import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) // TODO: tb-core ? -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + "'${queue.edqs.mode:null}'=='local'))") public @interface EdqsComponent { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java index 5055787fde..3e04dda3b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") public @interface InMemoryEdqsComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java index 9f112e7f59..9963972e9c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java @@ -21,7 +21,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + "'${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='kafka'))") public @interface KafkaEdqsComponent { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java index 91934293a6..b69aa52ec5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.edqs.EdqsService; @Service -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "false", matchIfMissing = true) +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "false", matchIfMissing = true) public class DummyEdqsService implements EdqsService { @Override From 3d9897f5af21ae26f197009cd6e985e5316f10ee Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 13:30:42 +0200 Subject: [PATCH 06/51] fixed NullPointer exception in CachedAttributesService --- .../server/dao/attributes/CachedAttributesService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index c5ac50c9f1..3c12ae3209 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -266,7 +266,9 @@ public class CachedAttributesService implements AttributesService { String key = keyVersionPair.getFirst(); Long version = keyVersionPair.getSecond(); cache.evict(new AttributeCacheKey(scope, entityId, key), version); - edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } return key; }, cacheExecutor)).collect(Collectors.toList())); } From 09bd3de984d15c7596bdc7d5152b3c5bc91ec342 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 16:42:31 +0200 Subject: [PATCH 07/51] test fixes --- .../controller/EntityQueryControllerTest.java | 16 ++-------------- .../discovery/HashPartitionServiceTest.java | 1 + .../state/DefaultDeviceStateServiceTest.java | 18 +++++++++--------- .../update/330/device_profile_001_out.json | 6 ++++-- 4 files changed, 16 insertions(+), 25 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 508f66e584..d9652946ab 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -817,26 +817,14 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(1, data.getTotalElements()); - Assert.assertEquals(1, data.getTotalPages()); - Assert.assertEquals(1, data.getData().size()); + findByQueryAndCheck(query, 1); // unnassign dashboard login(TENANT_EMAIL, TENANT_PASSWORD); doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD); - PageData dataAfterUnassign = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(0, dataAfterUnassign.getTotalElements()); - Assert.assertEquals(0, dataAfterUnassign.getTotalPages()); - Assert.assertEquals(0, dataAfterUnassign.getData().size()); + findByQueryAndCheck(query, 0); } private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 0a4009e8b2..0d3ae093aa 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -431,6 +431,7 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge"); ReflectionTestUtils.setField(partitionService, "edgePartitions", 10); + ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12); partitionService.init(); partitionService.partitionsInit(); return partitionService; diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 9880ec964b..3c3fe2716c 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -211,7 +211,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastConnectTime) @@ -298,7 +298,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastDisconnectTime) @@ -421,13 +421,13 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(lastInactivityTime) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -465,12 +465,12 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -1002,7 +1002,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.isActive()).isFalse(); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime) @@ -1170,7 +1170,7 @@ public class DefaultDeviceStateServiceTest { assertThat(attributeRequestCaptor.getAllValues()).hasSize(2) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { @@ -1179,7 +1179,7 @@ public class DefaultDeviceStateServiceTest { }); }) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { diff --git a/application/src/test/resources/update/330/device_profile_001_out.json b/application/src/test/resources/update/330/device_profile_001_out.json index 9a349c6638..29e2241ee9 100644 --- a/application/src/test/resources/update/330/device_profile_001_out.json +++ b/application/src/test/resources/update/330/device_profile_001_out.json @@ -64,7 +64,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } @@ -103,7 +104,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } From f128f49d44edbdafbe6927b3137a3106cdc33bf7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 18:57:23 +0200 Subject: [PATCH 08/51] test fixes --- .../service/entitiy/EntityServiceTest.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index fab4cba3f5..a559eca6ce 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -1178,8 +1178,7 @@ public class EntityServiceTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); + PageData result = findByQueryAndCheck(query, 1); String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName).isEqualTo(devices.get(0).getName()); @@ -1240,11 +1239,8 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setRootEntity(asset.getId()); EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - PageData relationsResult = entityService.findEntityDataByQuery(tenantId, customer.getId(), query); - long relationsResultCnt = entityService.countEntitiesByQuery(tenantId, customer.getId(), query); - - Assert.assertEquals(relationsCnt, relationsResult.getData().size()); - Assert.assertEquals(relationsCnt, relationsResultCnt); + findByQueryAndCheck(customer.getId(), query, relationsCnt); + countByQueryAndCheck(customer.getId(), query, relationsCnt); } } @@ -1497,9 +1493,6 @@ public class EntityServiceTest extends AbstractControllerTest { assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); } - private PageData searchEntities(EntityDataQuery query) { - return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - } private EntityDataQuery createDeviceSearchQuery(String deviceField, StringOperation operation, String searchQuery) { DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); @@ -1593,7 +1586,7 @@ public class EntityServiceTest extends AbstractControllerTest { .getLatest().get(currentAttributeKeyType).get("temperature").getValue()); } List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + assertThat(loadedTemperatures).containsExactlyInAnyOrderElementsOf(deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45); @@ -1614,8 +1607,7 @@ public class EntityServiceTest extends AbstractControllerTest { entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); - + assertThat(loadedHighTemperatures).containsExactlyInAnyOrderElementsOf(deviceHighTemperatures); } deviceService.deleteDevicesByTenantId(tenantId); } From bb7ea08e59f1c5300229eb92451532e37cb63751 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 14 Feb 2025 12:08:05 +0200 Subject: [PATCH 09/51] EDQS: refactoring, OOM handling, healthcheck --- .../TbRuleEngineQueueConsumerManager.java | 2 +- application/src/main/resources/logback.xml | 1 + .../EdqsEntityQueryControllerTest.java | 3 +- .../server/common/data/ObjectType.java | 8 +- common/edqs/pom.xml | 4 + .../server/edqs/processor/EdqsProcessor.java | 30 +++++-- .../server/edqs/processor/EdqsProducer.java | 2 +- .../server/edqs/repo/TenantRepo.java | 16 ++-- .../server/edqs/state/EdqsController.java | 40 +++++++++ .../server/edqs/state/EdqsStateService.java | 2 + .../edqs/state/KafkaEdqsStateService.java | 85 +++++++++++-------- .../edqs/state/LocalEdqsStateService.java | 12 ++- .../server/edqs/stats/EdqsStatsService.java | 6 +- .../consumer/MainQueueConsumerManager.java | 52 +++++++----- .../common/consumer/TbQueueConsumerTask.java | 2 +- .../kafka/TbKafkaConsumerStatsService.java | 13 +-- 16 files changed, 179 insertions(+), 99 deletions(-) create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 3636eb05af..1f421974be 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -72,7 +72,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager + diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index 9bb0bbe30a..1add0dae37 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -28,7 +28,6 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; import java.util.concurrent.TimeUnit; -import java.util.function.BiPredicate; import static org.awaitility.Awaitility.await; @@ -45,7 +44,7 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { @Autowired private EdqsService edqsService; - @MockBean + @MockBean // so that we don't do backup for tests private EdqsRocksDb edqsRocksDb; @Before diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index bc5ec58213..86252fb7f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -15,9 +15,8 @@ */ package org.thingsboard.server.common.data; -import java.util.Arrays; import java.util.EnumSet; -import java.util.HashSet; +import java.util.List; import java.util.Set; public enum ObjectType { @@ -68,12 +67,13 @@ public enum ObjectType { TENANT, TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS ); - public static final Set edqsTypes = new HashSet<>(edqsTenantTypes); + public static final Set edqsTypes = EnumSet.copyOf(edqsTenantTypes); public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); + public static final Set unversionedTypes = EnumSet.of(QUEUE_STATS); static { - edqsTypes.addAll(Arrays.asList(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + edqsTypes.addAll(List.of(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); } public EntityType toEntityType() { diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml index e58c7c97a0..2abd1286b8 100644 --- a/common/edqs/pom.xml +++ b/common/edqs/pom.xml @@ -68,6 +68,10 @@ org.thingsboard.common queue + + org.springframework.boot + spring-boot-starter-web + org.apache.kafka kafka-clients diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 6fe09f77d1..a90a029bc4 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -21,10 +21,12 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; @@ -72,6 +74,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; @EdqsComponent @@ -85,8 +88,8 @@ public class EdqsProcessor implements TbQueueHandler, private final EdqRepository repository; private final EdqsConfig config; private final EdqsPartitionService partitionService; - @Autowired - @Lazy + private final ConfigurableApplicationContext applicationContext; + @Autowired @Lazy private EdqsStateService stateService; private MainQueueConsumerManager, QueueConfig> eventsConsumer; @@ -96,17 +99,30 @@ public class EdqsProcessor implements TbQueueHandler, private ExecutorService mgmtExecutor; private ScheduledExecutorService scheduler; private ListeningExecutorService requestExecutor; + private ExecutorService repartitionExecutor; private final VersionsStore versionsStore = new VersionsStore(); private final AtomicInteger counter = new AtomicInteger(); // FIXME: TMP + @Getter + private Consumer errorHandler; + @PostConstruct private void init() { consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-mgmt"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); + repartitionExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-repartition")); + errorHandler = error -> { + if (error instanceof OutOfMemoryError) { + log.error("OOM detected, shutting down"); + repository.clear(); + Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-shutdown")) + .execute(applicationContext::close); + } + }; eventsConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) @@ -117,7 +133,7 @@ public class EdqsProcessor implements TbQueueHandler, ToEdqsMsg msg = queueMsg.getValue(); log.trace("Processing message: {}", msg); process(msg, EdqsQueue.EVENTS); - } catch (Throwable t) { + } catch (Exception t) { log.error("Failed to process message: {}", queueMsg, t); } } @@ -127,6 +143,7 @@ public class EdqsProcessor implements TbQueueHandler, .consumerExecutor(consumersExecutor) .taskExecutor(mgmtExecutor) .scheduler(scheduler) + .uncaughtErrorHandler(errorHandler) .build(); responseTemplate = queueFactory.createEdqsResponseTemplate(); } @@ -141,7 +158,7 @@ public class EdqsProcessor implements TbQueueHandler, if (event.getServiceType() != ServiceType.EDQS) { return; } - consumersExecutor.submit(() -> { + repartitionExecutor.submit(() -> { // todo: maybe cancel the task if new event comes try { Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); Set partitions = newPartitions.stream() @@ -220,8 +237,8 @@ public class EdqsProcessor implements TbQueueHandler, if (!versionsStore.isNew(key, version)) { return; } - } else { - log.warn("[{}] {} doesn't have version: {}", tenantId, objectType, edqsMsg); + } else if (!ObjectType.unversionedTypes.contains(objectType)) { + log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key); } if (queue != EdqsQueue.STATE) { stateService.save(tenantId, objectType, key, eventType, edqsMsg); @@ -272,6 +289,7 @@ public class EdqsProcessor implements TbQueueHandler, mgmtExecutor.shutdownNow(); scheduler.shutdownNow(); requestExecutor.shutdownNow(); + repartitionExecutor.shutdownNow(); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java index b21184110f..9b1e925367 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -53,7 +53,7 @@ public class EdqsProducer { TbQueueCallback callback = new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - log.debug("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); // fixme log levels + log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); // fixme log levels } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 68e1c42589..822061e918 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -153,7 +153,7 @@ public class TenantRepo { EntityData to = getOrCreate(entity.getTo()); boolean added = repo.add(from, to, TbStringPool.intern(entity.getType())); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); } } else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) { if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) { @@ -172,7 +172,7 @@ public class TenantRepo { if (relationsRepo != null) { boolean removed = relationsRepo.remove(entityRelation.getFrom().getId(), entityRelation.getTo().getId(), entityRelation.getType()); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); } } } else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) { @@ -223,7 +223,7 @@ public class TenantRepo { EntityData removed = getEntityMap(entityType).remove(entityId); if (removed != null) { getEntitySet(entityType).remove(removed); - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); UUID customerId = removed.getCustomerId(); if (customerId != null) { CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId); @@ -244,7 +244,7 @@ public class TenantRepo { Integer keyId = KeyDictionary.get(attributeKv.getKey()); boolean added = entityData.putAttr(keyId, attributeKv.getScope(), toDataPoint(attributeKv.getLastUpdateTs(), value)); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); } } } @@ -254,7 +254,7 @@ public class TenantRepo { if (entityData != null) { boolean removed = entityData.removeAttr(KeyDictionary.get(attributeKv.getKey()), attributeKv.getScope()); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); } } } @@ -266,7 +266,7 @@ public class TenantRepo { Integer keyId = KeyDictionary.get(latestTsKv.getKey()); boolean added = entityData.putTs(keyId, toDataPoint(latestTsKv.getTs(), value)); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); } } } @@ -276,7 +276,7 @@ public class TenantRepo { if (entityData != null) { boolean removed = entityData.removeTs(KeyDictionary.get(latestTsKv.getKey())); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); } } } @@ -331,7 +331,7 @@ public class TenantRepo { log.debug("[{}] Adding {} {}", tenantId, entityType, id); EntityData entityData = constructEntityData(entityType, entityId); getEntitySet(entityType).add(entityData); - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); return entityData; }); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java new file mode 100644 index 0000000000..721675245a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/edqs") +public class EdqsController { + + private final EdqsStateService edqsStateService; + + @GetMapping("/ready") + public ResponseEntity isReady() { + if (edqsStateService.isReady()) { + return ResponseEntity.ok().build(); + } else { + return ResponseEntity.badRequest().build(); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java index 8866a1c771..06d080ac64 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -29,4 +29,6 @@ public interface EdqsStateService { void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); + boolean isReady(); + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 84f3cfd22f..cfa831f720 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -42,10 +42,10 @@ import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.edqs.EdqsQueueFactory; import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; -import org.thingsboard.server.queue.util.AfterStartUp; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -66,6 +66,8 @@ public class KafkaEdqsStateService implements EdqsStateService { private QueueConsumerManager> eventsConsumer; private EdqsProducer stateProducer; + private boolean initialRestoreDone; + private ExecutorService consumersExecutor; private ExecutorService mgmtExecutor; private ScheduledExecutorService scheduler; @@ -76,7 +78,7 @@ public class KafkaEdqsStateService implements EdqsStateService { @PostConstruct private void init() { - consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-backup-consumer")); + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); @@ -92,8 +94,8 @@ public class KafkaEdqsStateService implements EdqsStateService { if (stateReadCount.incrementAndGet() % 100000 == 0) { log.info("[state] Processed {} msgs", stateReadCount.get()); } - } catch (Throwable t) { - log.error("Failed to process message: {}", queueMsg, t); + } catch (Exception e) { + log.error("Failed to process message: {}", queueMsg, e); // TODO: do something about the error - e.g. reprocess } } consumer.commit(); @@ -102,39 +104,48 @@ public class KafkaEdqsStateService implements EdqsStateService { .consumerExecutor(consumersExecutor) .taskExecutor(mgmtExecutor) .scheduler(scheduler) + .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) .build(); - eventsConsumer = QueueConsumerManager.>builder() // FIXME Slavik writes to the state while we read it, slows down the start + ExecutorService backupExecutor = ThingsBoardExecutors.newLimitedTasksExecutor(12, 1000, "events-to-backup-executor"); + eventsConsumer = QueueConsumerManager.>builder() // FIXME Slavik writes to the state while we read it, slows down the start. maybe start backup consumer after restore is finished .name("edqs-events-to-backup-consumer") .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer) -> { + CountDownLatch resultLatch = new CountDownLatch(msgs.size()); for (TbProtoQueueMsg queueMsg : msgs) { - try { - ToEdqsMsg msg = queueMsg.getValue(); - log.trace("Processing message: {}", msg); - - if (msg.hasEventMsg()) { - EdqsEventMsg eventMsg = msg.getEventMsg(); - String key = eventMsg.getKey(); - if (eventsReadCount.incrementAndGet() % 100000 == 0) { - log.info("[events-to-backup] Processed {} msgs", eventsReadCount.get()); - } - if (eventMsg.hasVersion()) { - if (!versionsStore.isNew(key, eventMsg.getVersion())) { - return; + backupExecutor.submit(() -> { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + + if (msg.hasEventMsg()) { + EdqsEventMsg eventMsg = msg.getEventMsg(); + String key = eventMsg.getKey(); + int count = eventsReadCount.incrementAndGet(); + if (count % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", count); + } + if (eventMsg.hasVersion()) { + if (!versionsStore.isNew(key, eventMsg.getVersion())) { + return; + } } - } - TenantId tenantId = getTenantId(msg); - ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); - EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); - log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); - stateProducer.send(tenantId, objectType, key, msg); + TenantId tenantId = getTenantId(msg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + stateProducer.send(tenantId, objectType, key, msg); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } finally { + resultLatch.countDown(); } - } catch (Throwable t) { - log.error("Failed to process message: {}", queueMsg, t); - } + }); } + resultLatch.await(); consumer.commit(); }) .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group @@ -149,22 +160,21 @@ public class KafkaEdqsStateService implements EdqsStateService { .build(); } - @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) - public void afterStartUp() { - eventsConsumer.subscribe(); - eventsConsumer.launch(); - } - @Override public void restore(Set partitions) { stateReadCount.set(0); //TODO Slavik: do not support remote mode in monolith setup long startTs = System.currentTimeMillis(); log.info("Restore started for partitions {}", partitions.stream().map(tpi -> tpi.getPartition().orElse(-1)).sorted().toList()); - stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update stateConsumer.awaitStop(0); // consumers should stop on their own because EdqsQueue.STATE.stopWhenRead is true, we just need to wait - log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), stateReadCount.get()); + + if (!initialRestoreDone) { + initialRestoreDone = true; + + eventsConsumer.subscribe(); + eventsConsumer.launch(); + } } @Override @@ -172,6 +182,11 @@ public class KafkaEdqsStateService implements EdqsStateService { // do nothing here, backup is done by events consumer } + @Override + public boolean isReady() { + return initialRestoreDone; + } + private TenantId getTenantId(ToEdqsMsg edqsMsg) { return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index 8d2d563d90..6a61488182 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -43,13 +43,11 @@ public class LocalEdqsStateService implements EdqsStateService { @Autowired private EdqsRocksDb db; - private Set partitions; + private boolean restoreDone; @Override public void restore(Set partitions) { - if (this.partitions == null) { - this.partitions = partitions; - } else { + if (restoreDone) { return; } @@ -62,6 +60,7 @@ public class LocalEdqsStateService implements EdqsStateService { log.error("[{}] Failed to restore value", key, e); } }); + restoreDone = true; } @Override @@ -78,4 +77,9 @@ public class LocalEdqsStateService implements EdqsStateService { } } + @Override + public boolean isReady() { + return restoreDone; + } + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java index 2ca036d23d..9619d086ad 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -52,9 +52,9 @@ public class EdqsStatsService { log.info("EDQS Stats: {}", values); } - public void reportTenantEdqsObject(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { + public void reportEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) - .reportEdqsObject(objectType, eventType); + .reportEvent(objectType, eventType); } @Getter @@ -78,7 +78,7 @@ public class EdqsStatsService { .collect(Collectors.joining(", ")); } - public void reportEdqsObject(ObjectType objectType, EdqsEventType eventType) { + public void reportEvent(ObjectType objectType, EdqsEventType eventType) { AtomicInteger objectCounter = getOrCreateObjectCounter(objectType); if (eventType == EdqsEventType.UPDATED){ objectCounter.incrementAndGet(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 2f44a9c4ae..9d45b168ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -39,6 +39,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.stream.Collectors; @Slf4j @@ -53,6 +54,7 @@ public class MainQueueConsumerManager uncaughtErrorHandler; private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); private final ReentrantLock lock = new ReentrantLock(); @@ -68,7 +70,8 @@ public class MainQueueConsumerManager> consumerCreator, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, - ExecutorService taskExecutor) { + ExecutorService taskExecutor, + Consumer uncaughtErrorHandler) { this.queueKey = queueKey; this.config = config; this.msgPackProcessor = msgPackProcessor; @@ -76,6 +79,7 @@ public class MainQueueConsumerManager consumerLoop = consumerExecutor.submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); - try { - consumerLoop(consumerTask.getConsumer()); - } catch (Throwable e) { - log.error("Failure in consumer loop", e); - } + consumerLoop(consumerTask.getConsumer()); log.info("[{}] Consumer stopped", consumerTask.getKey()); }); consumerTask.setTask(consumerLoop); } private void consumerLoop(TbQueueConsumer consumer) { - while (!stopped && !consumer.isStopped()) { - try { - List msgs = consumer.poll(config.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - processMsgs(msgs, consumer, config); - } catch (Exception e) { - if (!consumer.isStopped()) { - log.warn("Failed to process messages from queue", e); - try { - Thread.sleep(config.getPollInterval()); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + try { + while (!stopped && !consumer.isStopped()) { + try { + List msgs = consumer.poll(config.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + processMsgs(msgs, consumer, config); + } catch (Exception e) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", e); + try { + Thread.sleep(config.getPollInterval()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } - } - if (consumer.isStopped()) { + if (consumer.isStopped()) { + consumer.unsubscribe(); + } + } catch (Throwable t) { + log.error("Failure in consumer loop", t); + if (uncaughtErrorHandler != null) { + uncaughtErrorHandler.accept(t); + } consumer.unsubscribe(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java index 708e24fdac..4ed0ffa497 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java @@ -84,7 +84,7 @@ public class TbQueueConsumerTask { } log.trace("[{}] Awaited finish", key); } catch (Exception e) { - log.warn("[{}] Failed to await for consumer to stop", key, e); + log.warn("[{}] Failed to await for consumer to stop (timeout {} sec)", key, timeoutSec, e); } task = null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java index 0367a2be78..ade9204572 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java @@ -26,15 +26,10 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.discovery.PartitionService; import java.time.Duration; import java.util.ArrayList; @@ -56,10 +51,6 @@ public class TbKafkaConsumerStatsService { private final TbKafkaSettings kafkaSettings; private final TbKafkaConsumerStatisticConfig statsConfig; - @Lazy - @Autowired - private PartitionService partitionService; - private Consumer consumer; private ScheduledExecutorService statsPrintScheduler; @@ -111,9 +102,7 @@ public class TbKafkaConsumerStatsService { } private boolean isStatsPrintRequired() { - boolean isMyRuleEnginePartition = partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - boolean isMyCorePartition = partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - return log.isInfoEnabled() && (isMyRuleEnginePartition || isMyCorePartition); + return log.isInfoEnabled(); } private List getTopicsStatsWithLag(Map groupOffsets, Map endOffsets) { From 13fbcf4ba0fd5de64d6ec49fa9842973795384c5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 14 Feb 2025 12:31:36 +0200 Subject: [PATCH 10/51] Fix init for readiness probe controller --- .../java/org/thingsboard/server/edqs/state/EdqsController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java index 721675245a..06d5f89aa9 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.state; import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -23,6 +24,7 @@ import org.springframework.web.bind.annotation.RestController; @RestController @RequiredArgsConstructor +@ConditionalOnExpression("'${service.type:null}'=='edqs'") @RequestMapping("/api/edqs") public class EdqsController { From 52a6540e5a7346e01239ebf7a63cf1c9f15d23df Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 14 Feb 2025 15:12:00 +0200 Subject: [PATCH 11/51] added docker image for edqs --- edqs/src/main/conf/edqs.conf | 22 ++++ edqs/src/main/conf/logback.xml | 49 ++++++++ msa/edqs/docker/Dockerfile | 31 +++++ msa/edqs/docker/start-tb-edqs.sh | 31 +++++ msa/edqs/pom.xml | 190 +++++++++++++++++++++++++++++++ msa/pom.xml | 1 + 6 files changed, 324 insertions(+) create mode 100644 edqs/src/main/conf/edqs.conf create mode 100644 edqs/src/main/conf/logback.xml create mode 100644 msa/edqs/docker/Dockerfile create mode 100755 msa/edqs/docker/start-tb-edqs.sh create mode 100644 msa/edqs/pom.xml diff --git a/edqs/src/main/conf/edqs.conf b/edqs/src/main/conf/edqs.conf new file mode 100644 index 0000000000..ae880ef7e8 --- /dev/null +++ b/edqs/src/main/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/edqs/src/main/conf/logback.xml b/edqs/src/main/conf/logback.xml new file mode 100644 index 0000000000..4ff57d48ab --- /dev/null +++ b/edqs/src/main/conf/logback.xml @@ -0,0 +1,49 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + diff --git a/msa/edqs/docker/Dockerfile b/msa/edqs/docker/Dockerfile new file mode 100644 index 0000000000..e634029bbf --- /dev/null +++ b/msa/edqs/docker/Dockerfile @@ -0,0 +1,31 @@ +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk17:bookworm-slim + +COPY start-tb-edqs.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-edqs.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chown -R ${pkg.user}:${pkg.user} /tmp && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-edqs.sh"] \ No newline at end of file diff --git a/msa/edqs/docker/start-tb-edqs.sh b/msa/edqs/docker/start-tb-edqs.sh new file mode 100755 index 0000000000..bdd1ee48d3 --- /dev/null +++ b/msa/edqs/docker/start-tb-edqs.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER=${pkg.installFolder}/conf +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.edqs.ThingsboardEdqsApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=$CONF_FOLDER/logback.xml \ + org.springframework.boot.loader.launch.PropertiesLauncher diff --git a/msa/edqs/pom.xml b/msa/edqs/pom.xml new file mode 100644 index 0000000000..4874c06e4b --- /dev/null +++ b/msa/edqs/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + msa + + org.thingsboard.msa + edqs + pom + + ThingsBoard Entity Data Query Microservice + https://thingsboard.io + ThingsBoard Entity Data Query Microservice + + + UTF-8 + ${basedir}/../.. + edqs + edqs + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard + edqs + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-edqs + package + + copy + + + + + org.thingsboard + edqs + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/pom.xml b/msa/pom.xml index ff394295f4..259de5fb93 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -48,6 +48,7 @@ transport js-executor monitoring + edqs From fc5af455192ee59ba6526d3855d5d6882a745984 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 18 Feb 2025 17:01:39 +0200 Subject: [PATCH 12/51] EDQS: optimize attributes and latest kv --- .../server/service/edqs/EdqsSyncService.java | 2 +- ...faultTelemetrySubscriptionServiceTest.java | 2 +- .../server/common/data/edqs/AttributeKv.java | 6 +- .../server/common/data/edqs}/DataPoint.java | 2 +- .../server/common/data/edqs/LatestTsKv.java | 6 +- .../server/edqs/data/BaseEntityData.java | 2 +- .../server/edqs/data/DeviceData.java | 2 +- .../server/edqs/data/EntityData.java | 2 +- .../edqs/data/dp/AbstractDataPoint.java | 1 + .../edqs/data/dp/CompressedJsonDataPoint.java | 4 +- .../data/dp/CompressedStringDataPoint.java | 23 +-- .../server/edqs/load/TenantRepoLoader.java | 144 ------------------ .../server/edqs/processor/EdqsConverter.java | 96 +++++++++--- .../server/edqs/processor/EdqsProducer.java | 7 + .../server/edqs/repo/TenantRepo.java | 46 +----- .../server/edqs/util/RepositoryUtils.java | 2 +- common/proto/src/main/proto/queue.proto | 19 ++- 17 files changed, 125 insertions(+), 241 deletions(-) rename common/{edqs/src/main/java/org/thingsboard/server/edqs/data/dp => data/src/main/java/org/thingsboard/server/common/data/edqs}/DataPoint.java (95%) delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 80aeee8635..66f088a344 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -166,7 +166,7 @@ public abstract class EdqsSyncService { if (entityIdInfo != null) { process(entityIdInfo.tenantId(), RELATION, relation.toData()); } else { - log.info("Relation from entity not found: " + relation.getFromId()); + log.info("Relation from entity not found: " + relation.getFromType() + " " + relation.getFromId()); } } } diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 10fdd85504..677a414e2f 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -155,7 +155,7 @@ class DefaultTelemetrySubscriptionServiceTest { lenient().when(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).thenReturn(immediateFuture(Collections.emptyList())); // send partition change event so currentPartitions set is populated - telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)))); + telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)), Collections.emptyMap())); } @AfterEach diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java index f2c35466dc..9f65f72be4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java @@ -35,8 +35,10 @@ public class AttributeKv implements EdqsObject { private String key; private Long version; - private Long lastUpdateTs; // optional (on deletion) - private KvEntry value; // optional (on deletion) + private DataPoint dataPoint; // optional (on deletion) + + private Long lastUpdateTs; // only for serialization + private KvEntry value; // only for serialization public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) { this.entityId = entityId; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java similarity index 95% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java rename to common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java index 6cbdd35ad1..64b200fd45 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs.data.dp; +package org.thingsboard.server.common.data.edqs; import org.thingsboard.server.common.data.kv.DataType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java index 8aaabad1a7..19a767a320 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java @@ -33,8 +33,10 @@ public class LatestTsKv implements EdqsObject { private String key; private Long version; - private Long ts; // optional (on deletion) - private KvEntry value; // optional (on deletion) + private DataPoint dataPoint; // optional (on deletion) + + private Long ts; // only for serialization + private KvEntry value; // only for serialization public LatestTsKv(EntityId entityId, TsKvEntry tsKvEntry, Long version) { this.entityId = entityId; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java index 84785b8f3c..1789cd8ace 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.edqs.data.dp.BoolDataPoint; -import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.edqs.data.dp.LongDataPoint; import org.thingsboard.server.edqs.data.dp.StringDataPoint; import org.thingsboard.server.edqs.query.DataKey; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java index d416f86e92..f6794325d0 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.common.data.edqs.DataPoint; import java.util.Map; import java.util.UUID; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java index f13816976a..59e2c6c0c4 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java @@ -20,7 +20,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.edqs.query.DataKey; import org.thingsboard.server.edqs.repo.TenantRepo; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java index 91bc54e958..50740af6e1 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java @@ -17,6 +17,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.edqs.DataPoint; @RequiredArgsConstructor public abstract class AbstractDataPoint implements DataPoint { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java index 30f202181e..c1d1db94a8 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java @@ -19,8 +19,8 @@ import org.thingsboard.server.common.data.kv.DataType; public class CompressedJsonDataPoint extends CompressedStringDataPoint { - public CompressedJsonDataPoint(long ts, String value) { - super(ts, value); + public CompressedJsonDataPoint(long ts, byte[] compressedValue) { + super(ts, compressedValue); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java index 7056438338..0502bbccda 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -18,30 +18,18 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import lombok.SneakyThrows; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.edqs.repo.TbBytePool; import org.xerial.snappy.Snappy; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - public class CompressedStringDataPoint extends AbstractDataPoint { public static final int MIN_STR_SIZE_TO_COMPRESS = 512; @Getter - private final byte[] value; - - public static final AtomicInteger cnt = new AtomicInteger(); - public static final AtomicLong uncompressedLength = new AtomicLong(); - public static final AtomicLong compressedLength = new AtomicLong(); + private final byte[] compressedValue; @SneakyThrows - public CompressedStringDataPoint(long ts, String value) { + public CompressedStringDataPoint(long ts, byte[] compressedValue) { super(ts); - cnt.incrementAndGet(); - uncompressedLength.addAndGet(value.getBytes(StandardCharsets.UTF_8).length); - this.value = TbBytePool.intern(Snappy.compress(value)); - compressedLength.addAndGet(this.value.length); + this.compressedValue = compressedValue; } @Override @@ -52,13 +40,12 @@ public class CompressedStringDataPoint extends AbstractDataPoint { @SneakyThrows @Override public String getStr() { - return Snappy.uncompressString(value); + return Snappy.uncompressString(compressedValue); } - @SneakyThrows @Override public String valueToString() { - return Snappy.uncompressString(value); + return getStr(); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java deleted file mode 100644 index 6302e60f4c..0000000000 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.edqs.load; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.RandomStringUtils; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.edqs.AttributeKv; -import org.thingsboard.server.common.data.edqs.LatestTsKv; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.edqs.processor.EdqsConverter; -import org.thingsboard.server.edqs.repo.TenantRepo; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.UUID; - -@RequiredArgsConstructor -public class TenantRepoLoader { - - private static final int DEVICE_COUNT = 100000; - private static final int ATTRS_PER_DEVICE = 30; - private static final int ATTRS_AVG_STR_LENGTH = 12; - private static final int ATTRS_AVG_JSON_LENGTH = 265; - private static final int TS_PER_DEVICE = 29; - private static final int TS_AVG_STR_LENGTH = 59; - private static final int TS_AVG_JSON_LENGTH = 4005; - - private static final Map ATTR_CHANCES = new HashMap<>(); - private static final Random random = new Random(); - - static { - ATTR_CHANCES.put(DataType.BOOLEAN, 5); - ATTR_CHANCES.put(DataType.STRING, 49); - ATTR_CHANCES.put(DataType.LONG, 34); - ATTR_CHANCES.put(DataType.DOUBLE, 2); - ATTR_CHANCES.put(DataType.JSON, 10); - } - - private static final Map TS_CHANCES = new HashMap<>(); - - static { - TS_CHANCES.put(DataType.BOOLEAN, 6); - TS_CHANCES.put(DataType.STRING, 19); - TS_CHANCES.put(DataType.LONG, 36); - TS_CHANCES.put(DataType.DOUBLE, 32); - TS_CHANCES.put(DataType.JSON, 7); - } - - - @Getter - private final TenantRepo tenantRepo; - - public void load() { - long ts = System.currentTimeMillis() - DEVICE_COUNT; - for (int i = 0; i < DEVICE_COUNT; i++) { - DeviceId deviceId = new DeviceId(UUID.randomUUID()); - Device device = new Device(); - device.setId(deviceId); - device.setCreatedTime(ts + i); - device.setName("Device " + i); - device.setLabel("Device Label" + i); - device.setType("Device Type " + (i % 100)); - tenantRepo.addOrUpdate(EdqsConverter.toEntity(EntityType.DEVICE, device)); - for (int j = 0; j < ATTRS_PER_DEVICE; j++) { - String key = getRandomKey(); - AttributeKv attributeKv = new AttributeKv(); - attributeKv.setEntityId(deviceId); - attributeKv.setScope(AttributeScope.SERVER_SCOPE); - attributeKv.setKey(key); - attributeKv.setLastUpdateTs(ts); - attributeKv.setValue(getRandomKvEntry(key, ATTR_CHANCES, ATTRS_AVG_STR_LENGTH, ATTRS_AVG_JSON_LENGTH)); - tenantRepo.addOrUpdateAttribute(attributeKv); - } - for (int j = 0; j < TS_PER_DEVICE; j++) { - String key = getRandomKey(); - LatestTsKv latestTsKv = new LatestTsKv(); - latestTsKv.setEntityId(deviceId); - latestTsKv.setKey(key); - latestTsKv.setTs(ts); - latestTsKv.setValue(getRandomKvEntry(key, TS_CHANCES, TS_AVG_STR_LENGTH, TS_AVG_JSON_LENGTH)); - tenantRepo.addOrUpdateLatestKv(latestTsKv); - } - } - } - - private KvEntry getRandomKvEntry(String key, Map chances, int strLength, int jsnLength) { - int i = random.nextInt(100); - int s = 0; - for (var pair : chances.entrySet()) { - s += pair.getValue(); - if (i < s) { - switch (pair.getKey()) { - case BOOLEAN -> { - return new BooleanDataEntry(key, random.nextBoolean()); - } - case LONG -> { - return new LongDataEntry(key, random.nextLong()); - } - case DOUBLE -> { - return new DoubleDataEntry(key, random.nextDouble()); - } - case STRING -> { - return new StringDataEntry(key, StringUtils.randomAlphanumeric(strLength)); - } - case JSON -> { - return new JsonDataEntry(key, StringUtils.randomAlphanumeric(jsnLength)); - } - } - } - } - throw new RuntimeException("Something went wrong"); - } - - private String getRandomKey() { - return RandomStringUtils.randomAlphabetic(10); - } - -} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java index e4b9c2a35e..f5e405e516 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.edqs.processor; +import com.google.protobuf.ByteString; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.edqs.Entity; import org.thingsboard.server.common.data.edqs.LatestTsKv; @@ -31,15 +34,25 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedStringDataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.JsonDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.repo.TbBytePool; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; +import org.xerial.snappy.Snappy; import java.util.HashMap; import java.util.Map; import java.util.UUID; @Service +@Slf4j public class EdqsConverter { private final Map> converters = new HashMap<>(); @@ -50,7 +63,6 @@ public class EdqsConverter { converters.put(ObjectType.ATTRIBUTE_KV, new Converter() { @Override public byte[] serialize(ObjectType type, AttributeKv attributeKv) { - // TODO: some attributes may not fit into kafka var proto = TransportProtos.AttributeKvProto.newBuilder() .setEntityIdMSB(attributeKv.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(attributeKv.getEntityId().getId().getLeastSignificantBits()) @@ -58,11 +70,8 @@ public class EdqsConverter { .setScope(TransportProtos.AttributeScopeProto.forNumber(attributeKv.getScope().ordinal())) .setKey(attributeKv.getKey()) .setVersion(attributeKv.getVersion()); - if (attributeKv.getLastUpdateTs() != null) { - proto.setLastUpdateTs(attributeKv.getLastUpdateTs()); - } - if (attributeKv.getValue() != null) { - proto.setValue(KvProtoUtil.toKeyValueTypeProto(attributeKv.getValue())); + if (attributeKv.getLastUpdateTs() != null && attributeKv.getValue() != null) { + proto.setDataPoint(toDataPointProto(attributeKv.getLastUpdateTs(), attributeKv.getValue())); } return proto.build().toByteArray(); } @@ -73,14 +82,13 @@ public class EdqsConverter { EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); AttributeScope scope = AttributeScope.values()[proto.getScope().getNumber()]; - KvEntry value = proto.hasValue() ? KvProtoUtil.fromTsKvProto(proto.getValue()) : null; + DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; return AttributeKv.builder() .entityId(entityId) .scope(scope) .key(proto.getKey()) .version(proto.getVersion()) - .lastUpdateTs(proto.getLastUpdateTs()) - .value(value) + .dataPoint(dataPoint) .build(); } }); @@ -93,11 +101,8 @@ public class EdqsConverter { .setEntityType(ProtoUtils.toProto(latestTsKv.getEntityId().getEntityType())) .setKey(latestTsKv.getKey()) .setVersion(latestTsKv.getVersion()); - if (latestTsKv.getTs() != null) { - proto.setTs(latestTsKv.getTs()); - } - if (latestTsKv.getValue() != null) { - proto.setValue(KvProtoUtil.toKeyValueTypeProto(latestTsKv.getValue())); + if (latestTsKv.getTs() != null && latestTsKv.getValue() != null) { + proto.setDataPoint(toDataPointProto(latestTsKv.getTs(), latestTsKv.getValue())); } return proto.build().toByteArray(); } @@ -107,18 +112,73 @@ public class EdqsConverter { TransportProtos.LatestTsKvProto proto = TransportProtos.LatestTsKvProto.parseFrom(bytes); EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - KvEntry value = proto.hasValue() ? KvProtoUtil.fromTsKvProto(proto.getValue()) : null; + DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; return LatestTsKv.builder() .entityId(entityId) .key(proto.getKey()) - .ts(proto.getTs()) .version(proto.getVersion()) - .value(value) + .dataPoint(dataPoint) .build(); } }); } + public static DataPointProto toDataPointProto(long ts, KvEntry kvEntry) { + DataPointProto.Builder proto = DataPointProto.newBuilder(); + proto.setTs(ts); + switch (kvEntry.getDataType()) { + case BOOLEAN -> proto.setBoolV(kvEntry.getBooleanValue().get()); + case LONG -> proto.setLongV(kvEntry.getLongValue().get()); + case DOUBLE -> proto.setDoubleV(kvEntry.getDoubleValue().get()); + case STRING -> { + String strValue = kvEntry.getStrValue().get(); + if (strValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + proto.setStringV(strValue); + } else { + proto.setCompressedStringV(ByteString.copyFrom(compress(strValue))); + } + } + case JSON -> { + String jsonValue = kvEntry.getJsonValue().get(); + if (jsonValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + proto.setJsonV(jsonValue); + } else { + proto.setCompressedJsonV(ByteString.copyFrom(compress(jsonValue))); + } + } + } + return proto.build(); + } + + public static DataPoint fromDataPointProto(DataPointProto proto) { + long ts = proto.getTs(); + if (proto.hasBoolV()) { + return new BoolDataPoint(ts, proto.getBoolV()); + } else if (proto.hasLongV()) { + return new LongDataPoint(ts, proto.getLongV()); + } else if (proto.hasDoubleV()) { + return new DoubleDataPoint(ts, proto.getDoubleV()); + } else if (proto.hasStringV()) { + return new StringDataPoint(ts, proto.getStringV()); + } else if (proto.hasCompressedStringV()) { + return new CompressedStringDataPoint(ts, TbBytePool.intern(proto.getCompressedStringV().toByteArray())); + } else if (proto.hasJsonV()) { + return new JsonDataPoint(ts, proto.getJsonV()); + } else if (proto.hasCompressedJsonV()) { + return new CompressedJsonDataPoint(ts, TbBytePool.intern(proto.getCompressedJsonV().toByteArray())); + } else { + throw new IllegalArgumentException("Unsupported data point proto: " + proto); + } + } + + @SneakyThrows + private static byte[] compress(String value) { + byte[] compressed = Snappy.compress(value); + // TODO: limit the size + log.debug("Compressed {} bytes to {} bytes", value.length(), compressed.length); + return compressed; + } + public static Entity toEntity(EntityType entityType, Object entity) { Entity edqsEntity = new Entity(); edqsEntity.setType(entityType); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java index 9b1e925367..8973f3b5b8 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -17,6 +17,7 @@ package org.thingsboard.server.edqs.processor; import lombok.Builder; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.RecordTooLargeException; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -58,6 +59,12 @@ public class EdqsProducer { @Override public void onFailure(Throwable t) { + if (t instanceof RecordTooLargeException) { + if (!log.isDebugEnabled()) { + log.warn("[{}][{}][{}] Failed to publish msg to {}", tenantId, type, key, topic, t); // not logging the whole message + return; + } + } log.warn("[{}][{}][{}] Failed to publish msg to {}: {}", tenantId, type, key, topic, msg, t); } }; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 822061e918..9d6a4ab135 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.edqs.EdqsEvent; import org.thingsboard.server.common.data.edqs.EdqsEventType; import org.thingsboard.server.common.data.edqs.EdqsObject; @@ -30,7 +31,6 @@ import org.thingsboard.server.common.data.edqs.query.QueryResult; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityCountQuery; @@ -50,14 +50,6 @@ import org.thingsboard.server.edqs.data.EntityProfileData; import org.thingsboard.server.edqs.data.GenericData; import org.thingsboard.server.edqs.data.RelationsRepo; import org.thingsboard.server.edqs.data.TenantData; -import org.thingsboard.server.edqs.data.dp.BoolDataPoint; -import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; -import org.thingsboard.server.edqs.data.dp.CompressedStringDataPoint; -import org.thingsboard.server.edqs.data.dp.DataPoint; -import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; -import org.thingsboard.server.edqs.data.dp.JsonDataPoint; -import org.thingsboard.server.edqs.data.dp.LongDataPoint; -import org.thingsboard.server.edqs.data.dp.StringDataPoint; import org.thingsboard.server.edqs.query.EdqsDataQuery; import org.thingsboard.server.edqs.query.EdqsQuery; import org.thingsboard.server.edqs.query.SortableEntityData; @@ -240,9 +232,8 @@ public class TenantRepo { public void addOrUpdateAttribute(AttributeKv attributeKv) { var entityData = getOrCreate(attributeKv.getEntityId()); if (entityData != null) { - KvEntry value = attributeKv.getValue(); Integer keyId = KeyDictionary.get(attributeKv.getKey()); - boolean added = entityData.putAttr(keyId, attributeKv.getScope(), toDataPoint(attributeKv.getLastUpdateTs(), value)); + boolean added = entityData.putAttr(keyId, attributeKv.getScope(), attributeKv.getDataPoint()); if (added) { edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); } @@ -262,9 +253,8 @@ public class TenantRepo { public void addOrUpdateLatestKv(LatestTsKv latestTsKv) { var entityData = getOrCreate(latestTsKv.getEntityId()); if (entityData != null) { - KvEntry value = latestTsKv.getValue(); Integer keyId = KeyDictionary.get(latestTsKv.getKey()); - boolean added = entityData.putTs(keyId, toDataPoint(latestTsKv.getTs(), value)); + boolean added = entityData.putTs(keyId, latestTsKv.getDataPoint()); if (added) { edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); } @@ -281,42 +271,12 @@ public class TenantRepo { } } - private DataPoint toDataPoint(long ts, KvEntry kvEntry) { - return switch (kvEntry.getDataType()) { - case BOOLEAN -> new BoolDataPoint(ts, kvEntry.getBooleanValue().get()); - case STRING -> getStrDataPoint(ts, kvEntry.getStrValue().get()); - case LONG -> new LongDataPoint(ts, kvEntry.getLongValue().get()); - case DOUBLE -> new DoubleDataPoint(ts, kvEntry.getDoubleValue().get()); - case JSON -> getJsonDataPoint(ts, kvEntry.getJsonValue().get()); - }; - } - public void processFields(EntityFields fields) { if (fields instanceof AssetFields assetFields) { assetFields.setType(TbStringPool.intern(assetFields.getType())); } } - private static DataPoint getStrDataPoint(long ts, String strV) { - DataPoint dp; - if (strV.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { - dp = new StringDataPoint(ts, strV); - } else { - dp = new CompressedStringDataPoint(ts, strV); - } - return dp; - } - - private static DataPoint getJsonDataPoint(long ts, String strV) { - DataPoint dp; - if (strV.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { - dp = new JsonDataPoint(ts, strV); - } else { - dp = new CompressedJsonDataPoint(ts, strV); - } - return dp; - } - public ConcurrentMap> getEntityMap(EntityType entityType) { return entityMapByType.computeIfAbsent(entityType, et -> new ConcurrentHashMap<>()); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index b06db5519b..de54571d11 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -18,6 +18,7 @@ package org.thingsboard.server.edqs.util; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; @@ -41,7 +42,6 @@ import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.edqs.data.EntityData; -import org.thingsboard.server.edqs.data.dp.DataPoint; import org.thingsboard.server.edqs.query.DataKey; import org.thingsboard.server.edqs.query.EdqsCountQuery; import org.thingsboard.server.edqs.query.EdqsDataQuery; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index d21862d3e1..36c289b0ab 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -178,8 +178,7 @@ message AttributeKvProto { AttributeScopeProto scope = 4; string key = 5; int64 version = 6; - int64 lastUpdateTs = 7; - KeyValueProto value = 8; + DataPointProto dataPoint = 7; } message TsKvProto { @@ -193,9 +192,8 @@ message LatestTsKvProto { int64 entityIdLSB = 2; EntityTypeProto entityType = 3; string key = 4; - int64 ts = 5; - int64 version = 6; - KeyValueProto value = 7; + int64 version = 5; + DataPointProto dataPoint = 6; } message TsKvListProto { @@ -203,6 +201,17 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } +message DataPointProto { + int64 ts = 1; + optional bool boolV = 2; + optional int64 longV = 3; + optional double doubleV = 4; + optional string stringV = 5; + optional bytes compressedStringV = 6; + optional string jsonV = 7; + optional bytes compressedJsonV = 8; +} + message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; From 2a003709e0d93bfbb58ab0c20dac3c99841fe284 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Feb 2025 11:21:01 +0200 Subject: [PATCH 13/51] Fix tests --- .../server/service/edqs/DefaultEdqsService.java | 4 ++-- .../server/common/data/edqs/AttributeKv.java | 6 ++++++ .../server/common/data/edqs/EdqsObject.java | 4 ++++ .../thingsboard/server/common/data/edqs/Entity.java | 6 ++++++ .../server/common/data/edqs/LatestTsKv.java | 6 ++++++ .../server/common/data/relation/EntityRelation.java | 6 ++++++ .../server/edqs/processor/EdqsProcessor.java | 3 ++- .../server/edqs/processor/EdqsProducer.java | 2 +- .../edqs/{util => state}/EdqsPartitionService.java | 2 +- .../server/edqs/state/KafkaEdqsStateService.java | 1 - .../server/edqs/{processor => util}/EdqsConverter.java | 2 +- .../thingsboard/server/edqs/repo/AbstractEDQTest.java | 10 +++++++--- 12 files changed, 42 insertions(+), 10 deletions(-) rename common/edqs/src/main/java/org/thingsboard/server/edqs/{util => state}/EdqsPartitionService.java (97%) rename common/edqs/src/main/java/org/thingsboard/server/edqs/{processor => util}/EdqsConverter.java (99%) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index e7243ed3bc..5f30bc291a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -55,9 +55,9 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.edqs.processor.EdqsConverter; +import org.thingsboard.server.edqs.util.EdqsConverter; import org.thingsboard.server.edqs.processor.EdqsProducer; -import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.EdqsRequestMsg; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java index 9f65f72be4..c0e3abe9e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java @@ -20,6 +20,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; @@ -66,4 +67,9 @@ public class AttributeKv implements EdqsObject { return version; } + @Override + public ObjectType type() { + return ObjectType.ATTRIBUTE_KV; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java index 9a2836149a..d1e463443c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.edqs; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.thingsboard.server.common.data.ObjectType; public interface EdqsObject { @@ -25,4 +26,7 @@ public interface EdqsObject { @JsonIgnore Long version(); + @JsonIgnore + ObjectType type(); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java index decfbbd116..c08047c005 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.edqs.fields.EntityIdFields; @@ -59,4 +60,9 @@ public class Entity implements EdqsObject { return fields.getVersion(); } + @Override + public ObjectType type() { + return ObjectType.fromEntityType(type); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java index 19a767a320..b6a466df12 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -61,4 +62,9 @@ public class LatestTsKv implements EdqsObject { return version; } + @Override + public ObjectType type() { + return ObjectType.LATEST_TS_KV; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 927d592cd7..c04c490a26 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -25,6 +25,7 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.validation.Length; @@ -127,4 +128,9 @@ public class EntityRelation implements HasVersion, Serializable, EdqsObject { return version; } + @Override + public ObjectType type() { + return ObjectType.RELATION; + } + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index a90a029bc4..7e9d60d09a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -49,7 +49,8 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.edqs.repo.EdqRepository; import org.thingsboard.server.edqs.state.EdqsStateService; -import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java index 8973f3b5b8..bfd0d9df59 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -21,7 +21,7 @@ import org.apache.kafka.common.errors.RecordTooLargeException; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java similarity index 97% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java index 45de60687c..3d7757f17b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs.util; +package org.thingsboard.server.edqs.state; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index cfa831f720..6ab1fb9e1f 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -30,7 +30,6 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.edqs.processor.EdqsProcessor; import org.thingsboard.server.edqs.processor.EdqsProducer; -import org.thingsboard.server.edqs.util.EdqsPartitionService; import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java similarity index 99% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index f5e405e516..b7419451b9 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs.processor; +package org.thingsboard.server.edqs.util; import com.google.protobuf.ByteString; import lombok.RequiredArgsConstructor; diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java index 8fb680df1c..1f22f9d5ae 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -59,7 +59,7 @@ import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.edqs.processor.EdqsConverter; +import org.thingsboard.server.edqs.util.EdqsConverter; import java.util.Collections; import java.util.List; @@ -67,7 +67,7 @@ import java.util.UUID; @RunWith(SpringRunner.class) @Configuration -@ComponentScan({"org.thingsboard.server.edqs.repo"}) +@ComponentScan({"org.thingsboard.server.edqs.repo", "org.thingsboard.server.edqs.util"}) @EntityScan("org.thingsboard.server.edqs") @TestPropertySource(locations = {"classpath:edq-test.properties"}) @TestExecutionListeners({ @@ -77,6 +77,8 @@ public abstract class AbstractEDQTest { @Autowired protected InMemoryEdqRepository repository; + @Autowired + protected EdqsConverter edqsConverter; protected final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); protected final CustomerId customerId = new CustomerId(UUID.randomUUID()); @@ -218,7 +220,7 @@ public abstract class AbstractEDQTest { } protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, RelationTypeGroup group, String type) { - repository.get(tenantId).addOrUpdate(new EntityRelation(EntityIdFactory.getByTypeAndUuid(fromType, fromId), EntityIdFactory.getByTypeAndUuid(toType, toId), type, group)); + addOrUpdate(new EntityRelation(EntityIdFactory.getByTypeAndUuid(fromType, fromId), EntityIdFactory.getByTypeAndUuid(toType, toId), type, group)); } @@ -243,6 +245,8 @@ public abstract class AbstractEDQTest { } protected void addOrUpdate(EdqsObject edqsObject) { + byte[] serialized = edqsConverter.serialize(edqsObject.type(), edqsObject); + edqsObject = edqsConverter.deserialize(edqsObject.type(), serialized); repository.get(tenantId).addOrUpdate(edqsObject); } From 2580f4d3e5b758e4bc63ea3130fd1993de3b1b35 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 19 Feb 2025 12:05:05 +0200 Subject: [PATCH 14/51] fixed buildEntityDataQuery for AlarmDataQuery, key filtering for string null values --- .../server/service/subscription/TbAlarmDataSubCtx.java | 2 +- .../org/thingsboard/server/edqs/util/RepositoryUtils.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java index 70a259af6c..f5964c5d73 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java @@ -359,7 +359,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx { EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityDataSortOrder entitiesSortOrder; if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { - entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); + entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); } else { entitiesSortOrder = sortOrder; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index b06db5519b..e82c8d8b94 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -54,6 +54,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -66,10 +67,10 @@ import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.Co @Slf4j public class RepositoryUtils { - public static final Comparator SORT_ASC = Comparator.comparing(SortableEntityData::getSortValue) + public static final Comparator SORT_ASC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) .thenComparing(sp -> sp.getId().toString()); - public static final Comparator SORT_DESC = Comparator.comparing(SortableEntityData::getSortValue) + public static final Comparator SORT_DESC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) .thenComparing(sp -> sp.getId().toString()).reversed(); public static EntityType resolveEntityType(EntityFilter entityFilter) { @@ -210,7 +211,7 @@ public class RepositoryUtils { boolean checkResult = switch (valueType) { case STRING -> { String str = dp != null ? dp.valueToString() : null; - yield StringUtils.isEmpty(str) || checkKeyFilter(str, keyFilter.predicate()); + yield str != null && checkKeyFilter(str, keyFilter.predicate()); } case BOOLEAN -> { Boolean booleanValue = dp != null ? dp.getBool() : null; From 2431fcf6c5de78366ccb1b0cd4a45512fd046fa8 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 19 Feb 2025 12:18:44 +0200 Subject: [PATCH 15/51] fixed RepositoryUtilsTest --- .../org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index 48a72a9cc6..1562978a15 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -54,7 +54,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class RepositoryUtilsTest { private static Stream deviceNameFilters() { - return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), false), Arguments.of("loranet device 123", getNameFilter(StringOperation.STARTS_WITH, "lora"), true), Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "ra"), false), Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "123"), true), @@ -132,7 +132,7 @@ public class RepositoryUtilsTest { } private static Stream deviceNameComplexFilters() { - return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), false), Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false), Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "net")), true), From 46005a957a2f91e6a104d5b2459146f89d40c873 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 19 Feb 2025 14:46:39 +0200 Subject: [PATCH 16/51] fixed test --- .../thingsboard/server/service/edqs/EdqsSyncService.java | 2 +- .../server/common/data/edqs/fields/TenantFields.java | 5 +++++ .../org/thingsboard/server/edqs/util/RepositoryUtils.java | 6 ++++-- .../thingsboard/server/edqs/repo/RepositoryUtilsTest.java | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 80aeee8635..8bfa16c95a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -166,7 +166,7 @@ public abstract class EdqsSyncService { if (entityIdInfo != null) { process(entityIdInfo.tenantId(), RELATION, relation.toData()); } else { - log.info("Relation from entity not found: " + relation.getFromId()); + log.info("Relation from id not found: {} ", relation); } } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java index 6942d5ea7b..b86b36c1cd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java @@ -59,4 +59,9 @@ public class TenantFields extends AbstractEntityFields { public TenantFields(UUID id, Long version) { super(id, 0L, null, version); } + + @Override + public UUID getTenantId() { + return getId(); + } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index e82c8d8b94..331677e6b2 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -207,11 +207,13 @@ public class RepositoryUtils { default -> throw new IllegalStateException(); }; } - DataPoint dp = entity.getDataPoint(keyFilter.key(), null); + DataKey dataKey = keyFilter.key(); + DataPoint dp = entity.getDataPoint(dataKey, null); boolean checkResult = switch (valueType) { case STRING -> { String str = dp != null ? dp.valueToString() : null; - yield str != null && checkKeyFilter(str, keyFilter.predicate()); + yield (dataKey.type() == EntityKeyType.ENTITY_FIELD) ? (str == null || checkKeyFilter(str, keyFilter.predicate())) : + (str != null && checkKeyFilter(str, keyFilter.predicate())); } case BOOLEAN -> { Boolean booleanValue = dp != null ? dp.getBool() : null; diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index 1562978a15..48a72a9cc6 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -54,7 +54,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class RepositoryUtilsTest { private static Stream deviceNameFilters() { - return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), false), + return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), true), Arguments.of("loranet device 123", getNameFilter(StringOperation.STARTS_WITH, "lora"), true), Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "ra"), false), Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "123"), true), @@ -132,7 +132,7 @@ public class RepositoryUtilsTest { } private static Stream deviceNameComplexFilters() { - return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), false), + return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false), Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "net")), true), From 44721a255ca6dbda140067da2358808a0353a152 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 19 Feb 2025 15:24:22 +0200 Subject: [PATCH 17/51] added grafana dashboard for edqs entities --- .../dashboards/edqs_entities.json | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json diff --git a/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json new file mode 100644 index 0000000000..3e913d4856 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json @@ -0,0 +1,161 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "iteration": 1737564772936, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (objectType) (edqs_object_count{tenantId=~\"$tenantId\"})", + "interval": "", + "legendFormat": "{{objectType}}", + "refId": "A" + } + ], + "title": "EDQS object count", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(edqs_object_count, tenantId)", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": true, + "name": "tenantId", + "options": [], + "query": { + "query": "label_values(edqs_object_count, tenantId)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "EDQS", + "uid": "mK5A_DdHk", + "version": 9, + "weekStart": "" +} \ No newline at end of file From ded6daf2b3b01a463973218a2406fc8be86f722b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Feb 2025 15:44:05 +0200 Subject: [PATCH 18/51] Kafka states restore improvements --- .../TbRuleEngineQueueConsumerManager.java | 10 +-- .../AbstractTbQueueConsumerTemplate.java | 3 + .../consumer/MainQueueConsumerManager.java | 59 ++++++++++---- .../PartitionedQueueConsumerManager.java | 80 +++++++++++++++++++ .../common/consumer/QueueStateService.java | 76 ++++++++++++++++++ .../{QueueEvent.java => QueueTaskType.java} | 5 +- .../consumer/TbQueueConsumerManagerTask.java | 49 ++++++++---- .../common/consumer/TbQueueConsumerTask.java | 5 +- 8 files changed, 245 insertions(+), 42 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java rename common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/{QueueEvent.java => QueueTaskType.java} (83%) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 1f421974be..241aec25bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -33,14 +33,14 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.common.consumer.QueueEvent; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.DeleteQueueTask; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; @@ -78,13 +78,13 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager i @Override public void commit() { if (consumerLock.isLocked()) { + if (stopped) { + return; + } log.error("commit. consumerLock is locked. will wait with no timeout. it looks like a race conditions or deadlock topic " + topic, new RuntimeException("stacktrace")); } consumerLock.lock(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 9d45b168ee..7bd8076ce9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -23,6 +23,8 @@ import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdateConfigTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdatePartitionsTask; import org.thingsboard.server.queue.discovery.QueueKey; import java.util.Collection; @@ -31,6 +33,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; @@ -87,20 +90,24 @@ public class MainQueueConsumerManager createConsumerWrapper(C config) { if (config.isConsumerPerPartition()) { - this.consumerWrapper = new ConsumerPerPartitionWrapper(); + return new ConsumerPerPartitionWrapper(); } else { - this.consumerWrapper = new SingleConsumerWrapper(); + return new SingleConsumerWrapper(); } - log.debug("[{}] Initialized consumer for queue: {}", queueKey, config); } public void update(C config) { - addTask(TbQueueConsumerManagerTask.configUpdate(config)); + addTask(new UpdateConfigTask(config)); } public void update(Set partitions) { - addTask(TbQueueConsumerManagerTask.partitionChange(partitions)); + addTask(new UpdatePartitionsTask(partitions)); } protected void addTask(TbQueueConsumerManagerTask todo) { @@ -125,10 +132,10 @@ public class MainQueueConsumerManager partitions) { + private void doUpdate(Set partitions) { this.partitions = partitions; consumerWrapper.updatePartitions(partitions); } @@ -195,6 +202,15 @@ public class MainQueueConsumerManager consumerTask.awaitCompletion(timeoutSec)); log.debug("[{}] Unsubscribed and stopped consumers", queueKey); } - private static String partitionsToString(Collection partitions) { + static String partitionsToString(Collection partitions) { return partitions.stream().map(tpi -> tpi.getFullTopicName() + (tpi.isUseInternalPartition() ? "[" + tpi.getPartition().orElse(-1) + "]" : "")) .collect(Collectors.joining(", ", "[", "]")); @@ -279,15 +295,24 @@ public class MainQueueConsumerManager removedPartitions = new HashSet<>(consumers.keySet()); removedPartitions.removeAll(partitions); + log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions)); + removePartitions(removedPartitions); + addPartitions(addedPartitions, null); + } - removedPartitions.forEach((tpi) -> consumers.get(tpi).initiateStop()); - removedPartitions.forEach((tpi) -> consumers.remove(tpi).awaitCompletion()); + protected void removePartitions(Set removedPartitions) { + removedPartitions.forEach((tpi) -> Optional.ofNullable(consumers.get(tpi)).ifPresent(TbQueueConsumerTask::initiateStop)); + removedPartitions.forEach((tpi) -> Optional.ofNullable(consumers.remove(tpi)).ifPresent(TbQueueConsumerTask::awaitCompletion)); + } - addedPartitions.forEach((tpi) -> { + protected void addPartitions(Set partitions, Consumer onStop) { + partitions.forEach(tpi -> { Integer partitionId = tpi.getPartition().orElse(-1); String key = queueKey + "-" + partitionId; - TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> consumerCreator.apply(config, partitionId)); + Runnable callback = onStop != null ? () -> onStop.accept(tpi) : null; + + TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> consumerCreator.apply(config, partitionId), callback); consumers.put(tpi, consumer); consumer.subscribe(Set.of(tpi)); launchConsumer(consumer); @@ -316,7 +341,7 @@ public class MainQueueConsumerManager(queueKey, () -> consumerCreator.apply(config, null)); // no partitionId passed + consumer = new TbQueueConsumerTask<>(queueKey, () -> consumerCreator.apply(config, null), null); // no partitionId passed } consumer.subscribe(partitions); if (!consumer.isRunning()) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java new file mode 100644 index 0000000000..57f0950cdb --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common.consumer; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.AddPartitionsTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.RemovePartitionsTask; +import org.thingsboard.server.queue.discovery.QueueKey; + +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +@Slf4j +public class PartitionedQueueConsumerManager extends MainQueueConsumerManager { + + private final ConsumerPerPartitionWrapper consumerWrapper; + @Getter + private final String topic; + + @Builder(builderMethodName = "create") // not to conflict with super.builder() + public PartitionedQueueConsumerManager(QueueKey queueKey, String topic, long pollInterval, MsgPackProcessor msgPackProcessor, + BiFunction> consumerCreator, + ExecutorService consumerExecutor, ScheduledExecutorService scheduler, + ExecutorService taskExecutor, Consumer uncaughtErrorHandler) { + super(queueKey, QueueConfig.of(true, pollInterval), msgPackProcessor, consumerCreator, consumerExecutor, scheduler, taskExecutor, uncaughtErrorHandler); + this.topic = topic; + this.consumerWrapper = (ConsumerPerPartitionWrapper) super.consumerWrapper; + } + + @Override + public void update(Set partitions) { + throw new UnsupportedOperationException("Use manual addPartitions and removePartitions"); + } + + @Override + protected void processTask(TbQueueConsumerManagerTask task) { + if (task instanceof AddPartitionsTask addPartitionsTask) { + log.info("[{}] Added partitions: {}", queueKey, partitionsToString(addPartitionsTask.partitions())); + consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop()); + } else if (task instanceof RemovePartitionsTask removePartitionsTask) { + log.info("[{}] Removed partitions: {}", queueKey, partitionsToString(removePartitionsTask.partitions())); + consumerWrapper.removePartitions(removePartitionsTask.partitions()); + } + } + + public void addPartitions(Set partitions) { + addPartitions(partitions, null); + } + + public void addPartitions(Set partitions, Consumer onStop) { + addTask(new AddPartitionsTask(partitions, onStop)); + } + + public void removePartitions(Set partitions) { + addTask(new RemovePartitionsTask(partitions)); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java new file mode 100644 index 0000000000..bffe441f7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common.consumer; + +import lombok.Getter; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +public class QueueStateService { + + private PartitionedQueueConsumerManager stateConsumer; + private PartitionedQueueConsumerManager eventConsumer; + + @Getter + private Set partitions; + private final Lock lock = new ReentrantLock(); + + public void init(PartitionedQueueConsumerManager stateConsumer, PartitionedQueueConsumerManager eventConsumer) { + this.stateConsumer = stateConsumer; + this.eventConsumer = eventConsumer; + } + + public void update(Set newPartitions) { + lock.lock(); + Set oldPartitions = this.partitions != null ? this.partitions : Collections.emptySet(); + Set addedPartitions; + Set removedPartitions; + try { + addedPartitions = new HashSet<>(newPartitions); + addedPartitions.removeAll(oldPartitions); + removedPartitions = new HashSet<>(oldPartitions); + removedPartitions.removeAll(newPartitions); + this.partitions = newPartitions; + } finally { + lock.unlock(); + } + if (!removedPartitions.isEmpty()) { + stateConsumer.removePartitions(removedPartitions); + eventConsumer.removePartitions(removedPartitions.stream().map(tpi -> tpi.withTopic(eventConsumer.getTopic())).collect(Collectors.toSet())); + } + + if (!addedPartitions.isEmpty()) { + stateConsumer.addPartitions(addedPartitions, partition -> { + lock.lock(); + try { + if (this.partitions.contains(partition)) { + eventConsumer.addPartitions(Set.of(partition.withTopic(eventConsumer.getTopic()))); + } + } finally { + lock.unlock(); + } + }); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueTaskType.java similarity index 83% rename from common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueTaskType.java index 1a78cfc2ba..4e218cd268 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueTaskType.java @@ -17,8 +17,9 @@ package org.thingsboard.server.queue.common.consumer; import java.io.Serializable; -public enum QueueEvent implements Serializable { +public enum QueueTaskType implements Serializable { - PARTITION_CHANGE, CONFIG_UPDATE, DELETE + UPDATE_PARTITIONS, UPDATE_CONFIG, DELETE, + ADD_PARTITIONS, REMOVE_PARTITIONS } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java index 67bf370db0..7e2c848a66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java @@ -15,34 +15,49 @@ */ package org.thingsboard.server.queue.common.consumer; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import java.util.Set; +import java.util.function.Consumer; -@Getter -@ToString -@AllArgsConstructor -public class TbQueueConsumerManagerTask { +public interface TbQueueConsumerManagerTask { - private final QueueEvent event; - private QueueConfig config; - private Set partitions; - private boolean drainQueue; + QueueTaskType getType(); - public static TbQueueConsumerManagerTask delete(boolean drainQueue) { - return new TbQueueConsumerManagerTask(QueueEvent.DELETE, null, null, drainQueue); + record DeleteQueueTask(boolean drainQueue) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.DELETE; + } } - public static TbQueueConsumerManagerTask configUpdate(QueueConfig config) { - return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, config, null, false); + record UpdateConfigTask(QueueConfig config) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.UPDATE_CONFIG; + } } - public static TbQueueConsumerManagerTask partitionChange(Set partitions) { - return new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, null, partitions, false); + record UpdatePartitionsTask(Set partitions) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.UPDATE_PARTITIONS; + } + } + + record AddPartitionsTask(Set partitions, Consumer onStop) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.ADD_PARTITIONS; + } + } + + record RemovePartitionsTask(Set partitions) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.REMOVE_PARTITIONS; + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java index 4ed0ffa497..96085bf49a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java @@ -35,14 +35,17 @@ public class TbQueueConsumerTask { private final Object key; private volatile TbQueueConsumer consumer; private volatile Supplier> consumerSupplier; + @Getter + private final Runnable callback; @Setter private Future task; - public TbQueueConsumerTask(Object key, Supplier> consumerSupplier) { + public TbQueueConsumerTask(Object key, Supplier> consumerSupplier, Runnable callback) { this.key = key; this.consumer = null; this.consumerSupplier = consumerSupplier; + this.callback = callback; } public TbQueueConsumer getConsumer() { From 6bf56c8dc24d7e888e35d1a83069e07e862d728f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Feb 2025 15:54:18 +0200 Subject: [PATCH 19/51] EDQS repartitioning improvements --- .../server/edqs/processor/EdqsProcessor.java | 64 ++++++----- .../server/edqs/state/EdqsStateService.java | 2 +- .../edqs/state/KafkaEdqsStateService.java | 101 ++++++++---------- .../edqs/state/LocalEdqsStateService.java | 39 +++---- 4 files changed, 94 insertions(+), 112 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 7e9d60d09a..fa1309f932 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -43,14 +43,13 @@ import org.thingsboard.server.common.data.edqs.query.QueryResult; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.edqs.repo.EdqRepository; +import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.util.EdqsConverter; -import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; @@ -59,7 +58,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueHandler; import org.thingsboard.server.queue.TbQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.edqs.EdqsComponent; @@ -93,7 +92,8 @@ public class EdqsProcessor implements TbQueueHandler, @Autowired @Lazy private EdqsStateService stateService; - private MainQueueConsumerManager, QueueConfig> eventsConsumer; + @Getter + private PartitionedQueueConsumerManager> eventsConsumer; private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; private ExecutorService consumersExecutor; @@ -125,11 +125,15 @@ public class EdqsProcessor implements TbQueueHandler, } }; - eventsConsumer = MainQueueConsumerManager., QueueConfig>builder() + eventsConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) - .config(QueueConfig.of(true, config.getPollInterval())) + .topic(EdqsQueue.EVENTS.getTopic()) + .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { + if (consumer.isStopped()) { + return; + } try { ToEdqsMsg msg = queueMsg.getValue(); log.trace("Processing message: {}", msg); @@ -159,37 +163,31 @@ public class EdqsProcessor implements TbQueueHandler, if (event.getServiceType() != ServiceType.EDQS) { return; } - repartitionExecutor.submit(() -> { // todo: maybe cancel the task if new event comes - try { - Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); - Set partitions = newPartitions.stream() - .map(tpi -> tpi.withUseInternalPartition(true)) - .collect(Collectors.toSet()); + try { + Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); + Set partitions = newPartitions.stream() + .map(tpi -> tpi.withUseInternalPartition(true)) + .collect(Collectors.toSet()); - try { - stateService.restore(withTopic(partitions, EdqsQueue.STATE.getTopic())); // blocks until restored - } catch (Exception e) { - log.error("Failed to process restore for partitions {}", partitions, e); - } - eventsConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic())); - responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); + stateService.process(withTopic(partitions, EdqsQueue.STATE.getTopic())); + // eventsConsumer's partitions are updated by stateService + responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); - Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); - if (CollectionsUtil.isNotEmpty(oldPartitions)) { - Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() - .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); - if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { - log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); - } - repository.clearIf(tenantId -> { - Integer partition = partitionService.resolvePartition(tenantId); - return partition != null && removedPartitions.contains(partition); - }); + Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); + if (CollectionsUtil.isNotEmpty(oldPartitions)) { + Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() + .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); + if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); } - } catch (Throwable t) { - log.error("Failed to handle partition change event {}", event, t); + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId); + return partition != null && removedPartitions.contains(partition); + }); } - }); + } catch (Throwable t) { + log.error("Failed to handle partition change event {}", event, t); + } } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java index 06d080ac64..d45cc0de14 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -25,7 +25,7 @@ import java.util.Set; public interface EdqsStateService { - void restore(Set partitions); + void process(Set partitions); void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 6ab1fb9e1f..fc7f8d40d9 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -25,7 +25,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.edqs.processor.EdqsProcessor; @@ -34,8 +33,9 @@ import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueStateService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsQueue; @@ -44,7 +44,6 @@ import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; import java.util.Set; import java.util.UUID; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -54,19 +53,17 @@ import java.util.concurrent.atomic.AtomicInteger; @RequiredArgsConstructor @KafkaEdqsComponent @Slf4j -public class KafkaEdqsStateService implements EdqsStateService { +public class KafkaEdqsStateService extends QueueStateService, TbProtoQueueMsg> implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; private final EdqsQueueFactory queueFactory; private final EdqsProcessor edqsProcessor; - private MainQueueConsumerManager, QueueConfig> stateConsumer; - private QueueConsumerManager> eventsConsumer; + private PartitionedQueueConsumerManager> stateConsumer; + private QueueConsumerManager> eventsToBackupConsumer; private EdqsProducer stateProducer; - private boolean initialRestoreDone; - private ExecutorService consumersExecutor; private ExecutorService mgmtExecutor; private ScheduledExecutorService scheduler; @@ -81,11 +78,14 @@ public class KafkaEdqsStateService implements EdqsStateService { mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); - stateConsumer = MainQueueConsumerManager., QueueConfig>builder() // FIXME Slavik: if topic is empty + stateConsumer = PartitionedQueueConsumerManager.>create() // FIXME Slavik: if topic is empty .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) - .config(QueueConfig.of(true, config.getPollInterval())) + .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { + if (consumer.isStopped()) { + return; + } try { ToEdqsMsg msg = queueMsg.getValue(); log.trace("Processing message: {}", msg); @@ -94,7 +94,7 @@ public class KafkaEdqsStateService implements EdqsStateService { log.info("[state] Processed {} msgs", stateReadCount.get()); } } catch (Exception e) { - log.error("Failed to process message: {}", queueMsg, e); // TODO: do something about the error - e.g. reprocess + log.error("Failed to process message: {}", queueMsg, e); } } consumer.commit(); @@ -105,46 +105,43 @@ public class KafkaEdqsStateService implements EdqsStateService { .scheduler(scheduler) .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) .build(); + super.init(stateConsumer, edqsProcessor.getEventsConsumer()); - ExecutorService backupExecutor = ThingsBoardExecutors.newLimitedTasksExecutor(12, 1000, "events-to-backup-executor"); - eventsConsumer = QueueConsumerManager.>builder() // FIXME Slavik writes to the state while we read it, slows down the start. maybe start backup consumer after restore is finished + eventsToBackupConsumer = QueueConsumerManager.>builder() .name("edqs-events-to-backup-consumer") .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer) -> { - CountDownLatch resultLatch = new CountDownLatch(msgs.size()); for (TbProtoQueueMsg queueMsg : msgs) { - backupExecutor.submit(() -> { - try { - ToEdqsMsg msg = queueMsg.getValue(); - log.trace("Processing message: {}", msg); - - if (msg.hasEventMsg()) { - EdqsEventMsg eventMsg = msg.getEventMsg(); - String key = eventMsg.getKey(); - int count = eventsReadCount.incrementAndGet(); - if (count % 100000 == 0) { - log.info("[events-to-backup] Processed {} msgs", count); - } - if (eventMsg.hasVersion()) { - if (!versionsStore.isNew(key, eventMsg.getVersion())) { - return; - } - } + if (consumer.isStopped()) { + return; + } + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); - TenantId tenantId = getTenantId(msg); - ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); - EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); - log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); - stateProducer.send(tenantId, objectType, key, msg); + if (msg.hasEventMsg()) { + EdqsEventMsg eventMsg = msg.getEventMsg(); + String key = eventMsg.getKey(); + int count = eventsReadCount.incrementAndGet(); + if (count % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", count); + } + if (eventMsg.hasVersion()) { + if (!versionsStore.isNew(key, eventMsg.getVersion())) { + continue; + } } - } catch (Throwable t) { - log.error("Failed to process message: {}", queueMsg, t); - } finally { - resultLatch.countDown(); + + TenantId tenantId = getTenantId(msg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + stateProducer.send(tenantId, objectType, key, msg); } - }); + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } } - resultLatch.await(); consumer.commit(); }) .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group @@ -160,20 +157,12 @@ public class KafkaEdqsStateService implements EdqsStateService { } @Override - public void restore(Set partitions) { - stateReadCount.set(0); //TODO Slavik: do not support remote mode in monolith setup - long startTs = System.currentTimeMillis(); - log.info("Restore started for partitions {}", partitions.stream().map(tpi -> tpi.getPartition().orElse(-1)).sorted().toList()); - stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update - stateConsumer.awaitStop(0); // consumers should stop on their own because EdqsQueue.STATE.stopWhenRead is true, we just need to wait - log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), stateReadCount.get()); - - if (!initialRestoreDone) { - initialRestoreDone = true; - - eventsConsumer.subscribe(); - eventsConsumer.launch(); + public void process(Set partitions) { + if (getPartitions() == null) { + eventsToBackupConsumer.subscribe(); + eventsToBackupConsumer.launch(); } + super.update(partitions); } @Override @@ -194,7 +183,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private void preDestroy() { stateConsumer.stop(); stateConsumer.awaitStop(); - eventsConsumer.stop(); + eventsToBackupConsumer.stop(); stateProducer.stop(); consumersExecutor.shutdownNow(); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index 6a61488182..32f4159f9f 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -17,8 +17,6 @@ package org.thingsboard.server.edqs.state; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; @@ -38,29 +36,26 @@ import java.util.Set; @Slf4j public class LocalEdqsStateService implements EdqsStateService { - @Autowired @Lazy - private EdqsProcessor processor; - @Autowired - private EdqsRocksDb db; + private final EdqsProcessor processor; + private final EdqsRocksDb db; - private boolean restoreDone; + private Set partitions; @Override - public void restore(Set partitions) { - if (restoreDone) { - return; + public void process(Set partitions) { + if (this.partitions != null) { + db.forEach((key, value) -> { + try { + ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); + log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg); + processor.process(edqsMsg, EdqsQueue.STATE); + } catch (Exception e) { + log.error("[{}] Failed to restore value", key, e); + } + }); } - - db.forEach((key, value) -> { - try { - ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); - log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg); - processor.process(edqsMsg, EdqsQueue.STATE); - } catch (Exception e) { - log.error("[{}] Failed to restore value", key, e); - } - }); - restoreDone = true; + processor.getEventsConsumer().update(partitions); + this.partitions = partitions; } @Override @@ -79,7 +74,7 @@ public class LocalEdqsStateService implements EdqsStateService { @Override public boolean isReady() { - return restoreDone; + return partitions != null; } } From 7519889c1aef0044c49b7d186615bdb0aab4c8da Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 20 Feb 2025 12:25:08 +0200 Subject: [PATCH 20/51] Refactoring for EDQS repartitioning --- .../EdqsEntityQueryControllerTest.java | 4 +- .../server/edqs/processor/EdqsProcessor.java | 22 ++++---- .../server/edqs/state/EdqsController.java | 42 --------------- .../server/edqs/state/EdqsStateService.java | 6 ++- .../edqs/state/KafkaEdqsStateService.java | 53 ++++++------------- .../edqs/state/LocalEdqsStateService.java | 15 ++++-- .../common/msg/queue/TopicPartitionInfo.java | 2 +- .../consumer/MainQueueConsumerManager.java | 3 ++ .../PartitionedQueueConsumerManager.java | 5 -- .../common/consumer/QueueStateService.java | 8 ++- 10 files changed, 57 insertions(+), 103 deletions(-) delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index 1add0dae37..6833381f2d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -33,8 +33,8 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { -// "queue.type=kafka", // uncomment to use Kafka -// "queue.kafka.bootstrap.servers=10.7.1.254:9092", + "queue.type=kafka", // uncomment to use Kafka + "queue.kafka.bootstrap.servers=192.168.0.105:9092", "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index fa1309f932..968e637a3d 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -92,12 +92,11 @@ public class EdqsProcessor implements TbQueueHandler, @Autowired @Lazy private EdqsStateService stateService; - @Getter - private PartitionedQueueConsumerManager> eventsConsumer; + private PartitionedQueueConsumerManager> eventConsumer; private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; private ExecutorService consumersExecutor; - private ExecutorService mgmtExecutor; + private ExecutorService taskExecutor; private ScheduledExecutorService scheduler; private ListeningExecutorService requestExecutor; private ExecutorService repartitionExecutor; @@ -112,7 +111,7 @@ public class EdqsProcessor implements TbQueueHandler, @PostConstruct private void init() { consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); - mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-mgmt"); + taskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); repartitionExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-repartition")); @@ -125,7 +124,7 @@ public class EdqsProcessor implements TbQueueHandler, } }; - eventsConsumer = PartitionedQueueConsumerManager.>create() + eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) .topic(EdqsQueue.EVENTS.getTopic()) .pollInterval(config.getPollInterval()) @@ -146,10 +145,12 @@ public class EdqsProcessor implements TbQueueHandler, }) .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS)) .consumerExecutor(consumersExecutor) - .taskExecutor(mgmtExecutor) + .taskExecutor(taskExecutor) .scheduler(scheduler) .uncaughtErrorHandler(errorHandler) .build(); + stateService.init(eventConsumer); + responseTemplate = queueFactory.createEdqsResponseTemplate(); } @@ -171,7 +172,7 @@ public class EdqsProcessor implements TbQueueHandler, stateService.process(withTopic(partitions, EdqsQueue.STATE.getTopic())); // eventsConsumer's partitions are updated by stateService - responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); + responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); // FIXME: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); if (CollectionsUtil.isNotEmpty(oldPartitions)) { @@ -280,12 +281,13 @@ public class EdqsProcessor implements TbQueueHandler, @PreDestroy public void destroy() throws InterruptedException { - eventsConsumer.stop(); - eventsConsumer.awaitStop(); + eventConsumer.stop(); + eventConsumer.awaitStop(); responseTemplate.stop(); + stateService.stop(); consumersExecutor.shutdownNow(); - mgmtExecutor.shutdownNow(); + taskExecutor.shutdownNow(); scheduler.shutdownNow(); requestExecutor.shutdownNow(); repartitionExecutor.shutdownNow(); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java deleted file mode 100644 index 06d5f89aa9..0000000000 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.edqs.state; - -import lombok.RequiredArgsConstructor; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@RequiredArgsConstructor -@ConditionalOnExpression("'${service.type:null}'=='edqs'") -@RequestMapping("/api/edqs") -public class EdqsController { - - private final EdqsStateService edqsStateService; - - @GetMapping("/ready") - public ResponseEntity isReady() { - if (edqsStateService.isReady()) { - return ResponseEntity.ok().build(); - } else { - return ResponseEntity.badRequest().build(); - } - } - -} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java index d45cc0de14..1966618d4b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -20,15 +20,19 @@ import org.thingsboard.server.common.data.edqs.EdqsEventType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import java.util.Set; public interface EdqsStateService { + void init(PartitionedQueueConsumerManager> eventConsumer); + void process(Set partitions); void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); - boolean isReady(); + void stop(); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index fc7f8d40d9..f7382c9f15 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -15,13 +15,9 @@ */ package org.thingsboard.server.edqs.state; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; import org.thingsboard.server.common.data.id.TenantId; @@ -44,16 +40,13 @@ import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; @Service @RequiredArgsConstructor @KafkaEdqsComponent @Slf4j -public class KafkaEdqsStateService extends QueueStateService, TbProtoQueueMsg> implements EdqsStateService { +public class KafkaEdqsStateService implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; @@ -61,25 +54,19 @@ public class KafkaEdqsStateService extends QueueStateService> stateConsumer; + private QueueStateService, TbProtoQueueMsg> queueStateService; private QueueConsumerManager> eventsToBackupConsumer; private EdqsProducer stateProducer; - private ExecutorService consumersExecutor; - private ExecutorService mgmtExecutor; - private ScheduledExecutorService scheduler; - private final VersionsStore versionsStore = new VersionsStore(); private final AtomicInteger stateReadCount = new AtomicInteger(); private final AtomicInteger eventsReadCount = new AtomicInteger(); - @PostConstruct - private void init() { - consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); - mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); - scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); - - stateConsumer = PartitionedQueueConsumerManager.>create() // FIXME Slavik: if topic is empty + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) + .topic(EdqsQueue.STATE.getTopic()) .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { @@ -100,12 +87,13 @@ public class KafkaEdqsStateService extends QueueStateService queueFactory.createEdqsMsgConsumer(EdqsQueue.STATE)) - .consumerExecutor(consumersExecutor) - .taskExecutor(mgmtExecutor) - .scheduler(scheduler) + .consumerExecutor(eventConsumer.getConsumerExecutor()) + .taskExecutor(eventConsumer.getTaskExecutor()) + .scheduler(eventConsumer.getScheduler()) .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) .build(); - super.init(stateConsumer, edqsProcessor.getEventsConsumer()); + queueStateService = new QueueStateService<>(); + queueStateService.init(stateConsumer, eventConsumer); eventsToBackupConsumer = QueueConsumerManager.>builder() .name("edqs-events-to-backup-consumer") @@ -145,7 +133,7 @@ public class KafkaEdqsStateService extends QueueStateService queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group - .consumerExecutor(consumersExecutor) + .consumerExecutor(eventConsumer.getConsumerExecutor()) .threadPrefix("edqs-events-to-backup") .build(); @@ -158,11 +146,11 @@ public class KafkaEdqsStateService extends QueueStateService partitions) { - if (getPartitions() == null) { + if (queueStateService.getPartitions() == null) { eventsToBackupConsumer.subscribe(); eventsToBackupConsumer.launch(); } - super.update(partitions); + queueStateService.update(partitions); } @Override @@ -170,25 +158,16 @@ public class KafkaEdqsStateService extends QueueStateService> eventConsumer; private Set partitions; + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + this.eventConsumer = eventConsumer; + } + @Override public void process(Set partitions) { - if (this.partitions != null) { + if (this.partitions == null) { db.forEach((key, value) -> { try { ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); @@ -54,7 +62,7 @@ public class LocalEdqsStateService implements EdqsStateService { } }); } - processor.getEventsConsumer().update(partitions); + eventConsumer.update(partitions); this.partitions = partitions; } @@ -73,8 +81,7 @@ public class LocalEdqsStateService implements EdqsStateService { } @Override - public boolean isReady() { - return partitions != null; + public void stop() { } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 552bdf50d6..3bb489f209 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -58,7 +58,7 @@ public class TopicPartitionInfo { } public TopicPartitionInfo newByTopic(String topic) { - return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); } public String getTopic() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 7bd8076ce9..df683b94cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -54,8 +54,11 @@ public class MainQueueConsumerManager msgPackProcessor; protected final BiFunction> consumerCreator; + @Getter protected final ExecutorService consumerExecutor; + @Getter protected final ScheduledExecutorService scheduler; + @Getter protected final ExecutorService taskExecutor; protected final Consumer uncaughtErrorHandler; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java index 57f0950cdb..1b47d4c073 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -49,11 +49,6 @@ public class PartitionedQueueConsumerManager extends MainQ this.consumerWrapper = (ConsumerPerPartitionWrapper) super.consumerWrapper; } - @Override - public void update(Set partitions) { - throw new UnsupportedOperationException("Use manual addPartitions and removePartitions"); - } - @Override protected void processTask(TbQueueConsumerManagerTask task) { if (task instanceof AddPartitionsTask addPartitionsTask) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java index bffe441f7c..d022ad14de 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java @@ -41,6 +41,7 @@ public class QueueStateService { } public void update(Set newPartitions) { + newPartitions = withTopic(newPartitions, stateConsumer.getTopic()); lock.lock(); Set oldPartitions = this.partitions != null ? this.partitions : Collections.emptySet(); Set addedPartitions; @@ -54,9 +55,10 @@ public class QueueStateService { } finally { lock.unlock(); } + if (!removedPartitions.isEmpty()) { stateConsumer.removePartitions(removedPartitions); - eventConsumer.removePartitions(removedPartitions.stream().map(tpi -> tpi.withTopic(eventConsumer.getTopic())).collect(Collectors.toSet())); + eventConsumer.removePartitions(withTopic(removedPartitions, eventConsumer.getTopic())); } if (!addedPartitions.isEmpty()) { @@ -73,4 +75,8 @@ public class QueueStateService { } } + private Set withTopic(Set partitions, String topic) { + return partitions.stream().map(tpi -> tpi.withTopic(topic)).collect(Collectors.toSet()); + } + } From a48144d2bb6e49dbc69c5de0e852216423993a49 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 20 Feb 2025 17:34:28 +0200 Subject: [PATCH 21/51] EDQS code cleanup --- .../service/edqs/DefaultEdqsService.java | 1 - .../service/edqs/KafkaEdqsSyncService.java | 13 +- .../sync/tenant/TenantExportService.java | 224 ------------------ application/src/main/resources/logback.xml | 1 - .../src/main/resources/thingsboard.yml | 31 ++- .../EdqsEntityQueryControllerTest.java | 4 +- .../resources/application-test.properties | 2 +- .../src/test/resources/logback-test.xml | 2 +- .../server/edqs/util/EdqsRocksDb.java | 2 +- .../server/queue/edqs/EdqsConfig.java | 2 +- edqs/src/main/resources/edqs.yml | 22 +- 11 files changed, 48 insertions(+), 256 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 5f30bc291a..d5a65d248c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -125,7 +125,6 @@ public class DefaultEdqsService implements EdqsService { executor.submit(() -> { try { EdqsSyncState syncState = getSyncState(); - // FIXME: Slavik smart events check if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { processSystemRequest(ToCoreEdqsRequest.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index 0c6f948770..8dd4dbe4cb 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.edqs; -import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -30,18 +29,16 @@ import java.util.Collections; @ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") public class KafkaEdqsSyncService extends EdqsSyncService { - private final TbKafkaSettings kafkaSettings; - private TbKafkaAdmin kafkaAdmin; + private final boolean syncNeeded; - @PostConstruct - private void init() { - kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); + public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) { + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); + this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); } @Override public boolean isSyncNeeded() { - return kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); + return syncNeeded; } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java deleted file mode 100644 index 25ef73a081..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java +++ /dev/null @@ -1,224 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.tenant; - -import jakarta.annotation.PostConstruct; -import lombok.Data; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.ObjectType; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.audit.AuditLog; -import org.thingsboard.server.common.data.event.Event; -import org.thingsboard.server.common.data.event.EventType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.edqs.AttributeKv; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.edqs.LatestTsKv; -import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.dao.TenantEntityDao; -import org.thingsboard.server.dao.attributes.AttributesDao; -import org.thingsboard.server.dao.audit.AuditLogDao; -import org.thingsboard.server.dao.entity.EntityDaoRegistry; -import org.thingsboard.server.dao.event.EventDao; -import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.relation.RelationDao; -import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; - -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; - -import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; -import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; -import static org.thingsboard.server.common.data.ObjectType.EVENT; -import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; -import static org.thingsboard.server.common.data.ObjectType.RELATION; -import static org.thingsboard.server.common.data.ObjectType.TENANT; - -@Service -@RequiredArgsConstructor -@Slf4j -public class TenantExportService { - - private final EntityDaoRegistry entityDaoRegistry; - private final TenantDao tenantDao; - private final EventDao eventDao; - private final AuditLogDao auditLogDao; - private final AttributesDao attributesDao; - private final RelationDao relationDao; - private final TimeseriesLatestDao timeseriesLatestDao; - private final SqlPartitioningRepository partitioningRepository; - - private Map>> customExporters; - private Map relatedEntitiesExporters; - - private static final Set RELATED = EnumSet.of(EVENT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV); - - @PostConstruct - private void init() { - relatedEntitiesExporters = Map.of( - RELATION, this::exportRelations, - EVENT, this::exportEvents, // todo: query by tenant - ATTRIBUTE_KV, this::exportAttributes, - LATEST_TS_KV, this::exportLatestTelemetry - ); - customExporters = Map.of( - AUDIT_LOG, this::exportAuditLogs - ); - } - - public void exportTenant(TenantId tenantId, ExportConfig config, BiConsumer processor) { - log.info("[{}] Exporting tenant", tenantId); - Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); - if (tenant == null) { - throw new IllegalArgumentException("Tenant with id " + tenantId + " not found"); - } - - Set objectTypes = config.getIncludedObjectTypes(); - if (objectTypes.contains(TENANT)) { - exportEntity(tenantId, TENANT, tenant, config, processor); - } - - for (ObjectType type : objectTypes) { - if (RELATED.contains(type) || type == TENANT) { - continue; - } - log.debug("[{}] Exporting {} entities", tenantId, type); - if (!customExporters.containsKey(type)) { - TenantEntityDao dao = entityDaoRegistry.getTenantEntityDao(type); - var entities = new PageDataIterable<>(pageLink -> dao.findAllByTenantId(tenantId, pageLink), 100); - for (Object entity : entities) { - exportEntity(tenantId, type, entity, config, processor); - } - } else { - customExporters.get(type).accept(tenantId, processor); - } - } - } - - private void exportEntity(TenantId tenantId, ObjectType type, Object entity, ExportConfig config, BiConsumer processor) { - processor.accept(type, entity); - if (entity instanceof HasId hasId && hasId.getId() instanceof EntityId entityId) { - relatedEntitiesExporters.forEach((relatedEntityType, exporter) -> { - if (config.getIncludedObjectTypes().contains(relatedEntityType)) { - exporter.export(tenantId, entityId, processor); - } - }); - } - } - - private Map getPartitions(String table) { - List partitionsStartTime = partitioningRepository.fetchPartitions(table).stream().sorted().toList(); - if (partitionsStartTime.isEmpty()) { - return Collections.emptyMap(); - } - - Map partitions = new HashMap<>(); - for (int i = 0; i < partitionsStartTime.size(); i++) { - Long startTime = partitionsStartTime.get(i); - Long endTime; - if (partitionsStartTime.size() - 1 == i) { - endTime = System.currentTimeMillis(); - } else { - endTime = partitionsStartTime.get(i + 1) - 1; - } - partitions.put(startTime, endTime); - } - return partitions; - } - - private void exportAuditLogs(TenantId tenantId, BiConsumer processor) { - Map partitions = getPartitions(ModelConstants.AUDIT_LOG_TABLE_NAME); - partitions.forEach((startTime, endTime) -> { - PageDataIterable auditLogs = new PageDataIterable<>(pageLink -> { - return auditLogDao.findAuditLogsByTenantId(tenantId.getId(), null, new TimePageLink(pageLink, startTime, endTime)); - }, 512); - for (AuditLog auditLog : auditLogs) { - processor.accept(AUDIT_LOG, auditLog); - } - }); - } - - private void exportAttributes(TenantId tenantId, EntityId entityId, BiConsumer processor) { - for (AttributeScope attributeScope : AttributeScope.values()) { - List attributes = attributesDao.findAll(tenantId, entityId, attributeScope); - for (AttributeKvEntry entry : attributes) { - AttributeKv attributeKv = new AttributeKv(entityId, attributeScope, entry, entry.getVersion()); - processor.accept(ATTRIBUTE_KV, attributeKv); - } - } - } - - private void exportRelations(TenantId tenantId, EntityId entityId, BiConsumer processor) { - List relations = relationDao.findAllByFrom(tenantId, entityId); - for (EntityRelation relation : relations) { - processor.accept(RELATION, relation); - } - } - - @SneakyThrows - private void exportLatestTelemetry(TenantId tenantId, EntityId entityId, BiConsumer processor) { - List latestTelemetry = timeseriesLatestDao.findAllLatest(tenantId, entityId).get(30, TimeUnit.SECONDS); - for (TsKvEntry tsKvEntry : latestTelemetry) { - LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvEntry, tsKvEntry.getVersion()); - processor.accept(LATEST_TS_KV, latestTsKv); - } - } - - private void exportEvents(TenantId tenantId, EntityId entityId, BiConsumer processor) { - for (EventType eventType : EventType.values()) { - Map partitions = getPartitions(eventType.getTable()); - partitions.forEach((startTime, endTime) -> { - PageDataIterable events = new PageDataIterable<>(pageLink -> { - return eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, new TimePageLink(pageLink, startTime, endTime)); - }, 512); - for (Event event : events) { - processor.accept(EVENT, event); - } - }); - } - } - - private interface Exporter { - - void export(TenantId tenantId, EntityId entityId, BiConsumer processor); - - } - - @Data - public static class ExportConfig { - - private Set includedObjectTypes; - - } - -} diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 66d30777cd..7b71c1a272 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -28,7 +28,6 @@ - diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b20d6f11c3..21459814b7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1696,22 +1696,35 @@ queue: print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" edqs: sync: - enabled: "${TB_EDQS_SYNC_ENABLED:true}" # Enable/disable EDQS synchronization with postgres db FIXME: disable by default before release - entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" # batch size of entities being synced with EDQS - ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" # batch size of timeseries data being synced with EDQS - api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release - mode: "${TB_EDQS_MODE:local}" # local or remote + # Enable/disable EDQS synchronization FIXME: disable by default before release + enabled: "${TB_EDQS_SYNC_ENABLED:true}" + # Batch size of entities being synced with EDQS + entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" + # Batch size of timeseries data being synced with EDQS + ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" + # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) FIXME: disable by default before release + api_enabled: "${TB_EDQS_API_ENABLED:true}" + # Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices) + mode: "${TB_EDQS_MODE:local}" local: - rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:/tmp/edqs-backup}" + # Path to RocksDB for EDQS backup when running in local mode + rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:${user.home}/.rocksdb/edqs}" + # Number of partitions for EDQS topics partitions: "${TB_EDQS_PARTITIONS:12}" - partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data + # EDQS partitioning strategy: tenant (partition is resolved by tenant id) or none (no specific strategy, resolving by message key) + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS requests topic requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + # EDQS responses topic responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + # Poll interval for EDQS topics poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + # Maximum amount of pending requests to EDQS max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" - max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + # Maximum timeout for requests to EDQS + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" stats: - # Enable/disable statistics for EDQS service + # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" # Statistics printing interval for EDQS print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index 6833381f2d..efc290d3c5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -33,8 +33,8 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { - "queue.type=kafka", // uncomment to use Kafka - "queue.kafka.bootstrap.servers=192.168.0.105:9092", +// "queue.type=kafka", // uncomment to use Kafka +// "queue.kafka.bootstrap.servers=192.168.0.105:9092", "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 8933c36db5..bf90333c52 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -53,7 +53,7 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 -server.log_controller_error_stack_trace=true +server.log_controller_error_stack_trace=false transport.gateway.dashboard.sync.enabled=false diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index a0efcf52c1..13c93da411 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -9,7 +9,7 @@ - + diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java index 9b91b21c0d..eb3ef42f66 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java @@ -33,7 +33,7 @@ public class EdqsRocksDb extends TbRocksDb { @Getter private boolean isNew; - public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${java.io.tmpdir}/edqs-backup}") String path) { + public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${user.home}/.rocksdb/edqs}") String path) { super(path, new Options().setCreateIfMissing(true)); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java index f35827a8ce..8e3bd3b261 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -38,7 +38,7 @@ public class EdqsConfig { private long pollInterval; @Value("${queue.edqs.max_pending_requests:10000}") private int maxPendingRequests; - @Value("${queue.edqs.max_request_timeout:10000}") + @Value("${queue.edqs.max_request_timeout:20000}") private int maxRequestTimeout; public String getLabel() { diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index b67f2f5f9e..fa09da1cdd 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -47,19 +47,27 @@ spring: queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). - in_memory: - stats: - # For debug level - print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" edqs: - mode: "${TB_EDQS_MODE:local}" + # Number of partitions for EDQS topics partitions: "${TB_EDQS_PARTITIONS:12}" - partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data + # EDQS partitioning strategy: tenant (partitions are resolved and distributed by tenant id) or none (partitions are resolved by message key; each instance has all the partitions) + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS requests topic requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + # EDQS responses topic responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + # Poll interval for EDQS topics poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + # Maximum amount of pending requests to EDQS max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" - max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + # Maximum timeout for requests to EDQS + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + stats: + # Enable/disable statistics for EDQS + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" + kafka: # Kafka Bootstrap nodes in "host:port" format bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" From 347cb5bc368e344433a5474fde5f5c65205bc89c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 21 Feb 2025 11:48:21 +0200 Subject: [PATCH 22/51] EDQS code cleanup --- .../service/edqs/DefaultEdqsService.java | 3 + .../service/edqs/KafkaEdqsSyncService.java | 2 - .../server/common/data/ObjectType.java | 9 ++- .../server/common/data/alarm/AlarmType.java | 27 -------- .../data/edqs/fields/GenericFields.java | 16 +---- .../common/data/edqs/fields/TenantFields.java | 3 - .../data/page/BasePageDataIterable.java | 9 +-- .../common/data/page/PageDataIterable.java | 5 -- .../common/data/query/EntityFilter.java | 3 +- .../data/dp/CompressedStringDataPoint.java | 3 +- .../server/edqs/data/dp/JsonDataPoint.java | 2 +- .../server/edqs/data/dp/StringDataPoint.java | 2 +- .../server/edqs/processor/EdqsProcessor.java | 63 ++++++++++-------- .../server/edqs/processor/EdqsProducer.java | 10 +-- ...sitory.java => DefaultEdqsRepository.java} | 16 ++--- ...EdqRepository.java => EdqsRepository.java} | 5 +- .../server/edqs/repo/TenantRepo.java | 1 + .../edqs/state/KafkaEdqsStateService.java | 4 +- .../server/edqs/util/EdqsConverter.java | 5 +- .../edqs/{repo => util}/TbBytePool.java | 2 +- .../edqs/{repo => util}/TbStringPool.java | 2 +- .../queue/edqs/KafkaEdqsQueueFactory.java | 15 +++-- .../provider/KafkaMonolithQueueFactory.java | 2 +- .../provider/KafkaTbCoreQueueFactory.java | 2 +- .../common/util/ExceptionUtil.java | 10 +++ .../server/dao/TenantEntityDao.java | 5 -- .../server/dao/entity/EntityDaoRegistry.java | 18 +---- .../server/dao/model/sql/AlarmTypeEntity.java | 65 ------------------- .../dao/sql/alarm/AlarmTypeRepository.java | 30 --------- .../dao/sql/alarm/JpaAlarmCommentDao.java | 6 -- .../server/dao/sql/alarm/JpaAlarmDao.java | 6 -- .../server/dao/sql/alarm/JpaAlarmTypeDao.java | 46 ------------- .../dao/sql/alarm/JpaEntityAlarmDao.java | 6 -- .../server/dao/sql/asset/JpaAssetDao.java | 6 -- .../dao/sql/asset/JpaAssetProfileDao.java | 6 -- .../dao/sql/customer/JpaCustomerDao.java | 6 -- .../dao/sql/dashboard/JpaDashboardDao.java | 6 -- .../sql/device/DeviceProfileRepository.java | 1 - .../sql/device/JpaDeviceCredentialsDao.java | 6 -- .../server/dao/sql/device/JpaDeviceDao.java | 6 -- .../dao/sql/device/JpaDeviceProfileDao.java | 6 -- .../server/dao/sql/edge/JpaEdgeDao.java | 6 -- .../sql/entityview/EntityViewRepository.java | 1 - .../dao/sql/entityview/JpaEntityViewDao.java | 6 -- .../notification/JpaNotificationRuleDao.java | 6 -- .../JpaNotificationTargetDao.java | 6 -- .../JpaNotificationTemplateDao.java | 6 -- .../server/dao/sql/ota/JpaOtaPackageDao.java | 6 -- .../server/dao/sql/queue/JpaQueueDao.java | 6 -- .../dao/sql/queue/JpaQueueStatsDao.java | 6 -- .../dao/sql/resource/JpaTbResourceDao.java | 6 -- .../server/dao/sql/rpc/JpaRpcDao.java | 6 -- .../server/dao/sql/rule/JpaRuleChainDao.java | 6 -- .../server/dao/sql/rule/JpaRuleNodeDao.java | 6 -- .../dao/sql/settings/JpaAdminSettingsDao.java | 6 -- .../sql/usagerecord/JpaApiUsageStateDao.java | 6 -- .../dao/sql/user/JpaUserAuthSettingsDao.java | 6 -- .../dao/sql/user/JpaUserCredentialsDao.java | 6 -- .../server/dao/sql/user/JpaUserDao.java | 6 -- .../dao/sql/user/JpaUserSettingsDao.java | 6 -- .../dao/sql/widget/JpaWidgetTypeDao.java | 6 -- .../dao/sql/widget/JpaWidgetsBundleDao.java | 6 -- .../dao/service/EntityDaoRegistryTest.java | 31 --------- edqs/src/main/resources/logback.xml | 1 - .../server/edqs/repo/AbstractEDQTest.java | 4 +- ...q-test.properties => edqs-test.properties} | 0 66 files changed, 99 insertions(+), 504 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java rename common/edqs/src/main/java/org/thingsboard/server/edqs/repo/{InMemoryEdqRepository.java => DefaultEdqsRepository.java} (89%) rename common/edqs/src/main/java/org/thingsboard/server/edqs/repo/{EdqRepository.java => EdqsRepository.java} (92%) rename common/edqs/src/main/java/org/thingsboard/server/edqs/{repo => util}/TbBytePool.java (96%) rename common/edqs/src/main/java/org/thingsboard/server/edqs/{repo => util}/TbStringPool.java (96%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java rename edqs/src/test/resources/{edq-test.properties => edqs-test.properties} (100%) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index d5a65d248c..a6a9b9bb23 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -68,6 +68,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.environment.DistributedLock; import org.thingsboard.server.queue.environment.DistributedLockService; @@ -90,6 +91,7 @@ public class DefaultEdqsService implements EdqsService { private final DistributedLockService distributedLockService; private final AttributesService attributesService; private final EdqsPartitionService edqsPartitionService; + private final TopicService topicService; @Autowired @Lazy private TbClusterService clusterService; @Autowired @Lazy @@ -109,6 +111,7 @@ public class DefaultEdqsService implements EdqsService { eventsProducer = EdqsProducer.builder() .queue(EdqsQueue.EVENTS) .partitionService(edqsPartitionService) + .topicService(topicService) .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) .build(); if (apiEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index 8dd4dbe4cb..7dc25b92c8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.edqs; -import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.queue.edqs.EdqsQueue; @@ -25,7 +24,6 @@ import org.thingsboard.server.queue.kafka.TbKafkaSettings; import java.util.Collections; @Service -@RequiredArgsConstructor @ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") public class KafkaEdqsSyncService extends EdqsSyncService { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index 86252fb7f8..8fa08d38e9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -23,7 +23,6 @@ public enum ObjectType { TENANT, TENANT_PROFILE, CUSTOMER, - ADMIN_SETTINGS, QUEUE, RPC, RULE_CHAIN, @@ -32,8 +31,6 @@ public enum ObjectType { EVENT, RULE_NODE, USER, - USER_CREDENTIALS, - USER_AUTH_SETTINGS, EDGE, WIDGETS_BUNDLE, WIDGET_TYPE, @@ -54,7 +51,6 @@ public enum ObjectType { NOTIFICATION_TEMPLATE, NOTIFICATION_RULE, ALARM_COMMENT, - ALARM_TYPE, API_USAGE_STATE, QUEUE_STATS, @@ -70,7 +66,10 @@ public enum ObjectType { public static final Set edqsTypes = EnumSet.copyOf(edqsTenantTypes); public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); - public static final Set unversionedTypes = EnumSet.of(QUEUE_STATS); + public static final Set unversionedTypes = EnumSet.of( + QUEUE_STATS, // created once, never updated + TENANT_PROFILE // only for total count calculation + ); static { edqsTypes.addAll(List.of(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java deleted file mode 100644 index 0813e72123..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.alarm; - -import lombok.Data; -import org.thingsboard.server.common.data.id.TenantId; - -@Data -public class AlarmType { - - private TenantId tenantId; - private String type; - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java index ff3af4ee65..94e8e35dcc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java @@ -15,27 +15,15 @@ */ package org.thingsboard.server.common.data.edqs.fields; -import com.fasterxml.jackson.databind.JsonNode; -import lombok.Data; import lombok.NoArgsConstructor; -import lombok.experimental.SuperBuilder; -import java.util.UUID; -import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; +import java.util.UUID; -@Data @NoArgsConstructor -@SuperBuilder public class GenericFields extends AbstractEntityFields { - private String additionalInfo; - - public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) { - super(id, createdTime, tenantId, name, version); - this.additionalInfo = getText(additionalInfo); - } - public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { super(id, createdTime, tenantId, name, version); } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java index 6942d5ea7b..c11c221a9b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java @@ -56,7 +56,4 @@ public class TenantFields extends AbstractEntityFields { this.region = region; } - public TenantFields(UUID id, Long version) { - super(id, 0L, null, version); - } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java index 625a1d417c..8d46a46781 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java @@ -23,7 +23,6 @@ import java.util.NoSuchElementException; public abstract class BasePageDataIterable implements Iterable, Iterator { private final int fetchSize; - private SortOrder sortOrder; private List currentItems; private int currentIdx; @@ -36,12 +35,6 @@ public abstract class BasePageDataIterable implements Iterable, Iterator iterator() { return this; @@ -50,7 +43,7 @@ public abstract class BasePageDataIterable implements Iterable, Iterator extends BasePageDataIterable { this.function = function; } - public PageDataIterable(FetchFunction function, int fetchSize, SortOrder sortOrder) { - super(fetchSize, sortOrder); - this.function = function; - } - @Override PageData fetchPageData(PageLink link) { return function.fetch(link); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java index f881ed15c3..8e45e20efa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java @@ -39,7 +39,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), @JsonSubTypes.Type(value = EntityViewSearchQueryFilter.class, name = "entityViewSearchQuery"), - @JsonSubTypes.Type(value = EdgeSearchQueryFilter.class, name = "edgeSearchQuery")}) + @JsonSubTypes.Type(value = EdgeSearchQueryFilter.class, name = "edgeSearchQuery") +}) public interface EntityFilter { @JsonIgnore diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java index 0502bbccda..0ef614de06 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -18,6 +18,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import lombok.SneakyThrows; import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.util.TbBytePool; import org.xerial.snappy.Snappy; public class CompressedStringDataPoint extends AbstractDataPoint { @@ -29,7 +30,7 @@ public class CompressedStringDataPoint extends AbstractDataPoint { @SneakyThrows public CompressedStringDataPoint(long ts, byte[] compressedValue) { super(ts); - this.compressedValue = compressedValue; + this.compressedValue = TbBytePool.intern(compressedValue); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java index 05cafa3edb..739cd1902e 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java @@ -17,7 +17,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.edqs.repo.TbStringPool; +import org.thingsboard.server.edqs.util.TbStringPool; public class JsonDataPoint extends AbstractDataPoint { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java index fabf15a359..281c467c31 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java @@ -17,7 +17,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.edqs.repo.TbStringPool; +import org.thingsboard.server.edqs.util.TbStringPool; public class StringDataPoint extends AbstractDataPoint { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 968e637a3d..42e00be7fc 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -24,12 +24,12 @@ import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ExceptionUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -46,7 +46,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.edqs.repo.EdqRepository; +import org.thingsboard.server.edqs.repo.EdqsRepository; import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.util.EdqsConverter; @@ -68,6 +68,7 @@ import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.edqs.EdqsQueueFactory; import org.thingsboard.server.queue.util.AfterStartUp; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutorService; @@ -85,7 +86,7 @@ public class EdqsProcessor implements TbQueueHandler, private final EdqsQueueFactory queueFactory; private final EdqsConverter converter; - private final EdqRepository repository; + private final EdqsRepository repository; private final EdqsConfig config; private final EdqsPartitionService partitionService; private final ConfigurableApplicationContext applicationContext; @@ -99,11 +100,10 @@ public class EdqsProcessor implements TbQueueHandler, private ExecutorService taskExecutor; private ScheduledExecutorService scheduler; private ListeningExecutorService requestExecutor; - private ExecutorService repartitionExecutor; private final VersionsStore versionsStore = new VersionsStore(); - private final AtomicInteger counter = new AtomicInteger(); // FIXME: TMP + private final AtomicInteger counter = new AtomicInteger(); @Getter private Consumer errorHandler; @@ -114,7 +114,6 @@ public class EdqsProcessor implements TbQueueHandler, taskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); - repartitionExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-repartition")); errorHandler = error -> { if (error instanceof OutOfMemoryError) { log.error("OOM detected, shutting down"); @@ -135,7 +134,6 @@ public class EdqsProcessor implements TbQueueHandler, } try { ToEdqsMsg msg = queueMsg.getValue(); - log.trace("Processing message: {}", msg); process(msg, EdqsQueue.EVENTS); } catch (Exception t) { log.error("Failed to process message: {}", queueMsg, t); @@ -195,27 +193,19 @@ public class EdqsProcessor implements TbQueueHandler, public ListenableFuture> handle(TbProtoQueueMsg queueMsg) { ToEdqsMsg toEdqsMsg = queueMsg.getValue(); return requestExecutor.submit(() -> { - EdqsResponse response = new EdqsResponse(); + EdqsRequest request; + TenantId tenantId; + CustomerId customerId; try { - EdqsRequest request = JacksonUtil.fromString(toEdqsMsg.getRequestMsg().getValue(), EdqsRequest.class); - TenantId tenantId = getTenantId(toEdqsMsg); - CustomerId customerId = getCustomerId(toEdqsMsg); - log.info("[{}] Handling request: {}", tenantId, request); - - if (request.getEntityDataQuery() != null) { - PageData result = repository.findEntityDataByQuery(tenantId, customerId, - request.getEntityDataQuery(), false); - response.setEntityDataQueryResult(result.mapData(QueryResult::toOldEntityData)); - } else if (request.getEntityCountQuery() != null) { - long result = repository.countEntitiesByQuery(tenantId, customerId, request.getEntityCountQuery(), tenantId.isSysTenantId()); - response.setEntityCountQueryResult(result); - } - - log.info("Answering with response: {}", response); - } catch (Throwable e) { - response.setError(ExceptionUtils.getStackTrace(e)); // TODO: return only the message - log.info("Answering with error", e); + request = Objects.requireNonNull(JacksonUtil.fromString(toEdqsMsg.getRequestMsg().getValue(), EdqsRequest.class)); + tenantId = getTenantId(toEdqsMsg); + customerId = getCustomerId(toEdqsMsg); + } catch (Exception e) { + log.error("Failed to parse request msg: {}", toEdqsMsg, e); + throw e; } + + EdqsResponse response = processRequest(tenantId, customerId, request); return new TbProtoQueueMsg<>(queueMsg.getKey(), FromEdqsMsg.newBuilder() .setResponseMsg(TransportProtos.EdqsResponseMsg.newBuilder() .setValue(JacksonUtil.toString(response)) @@ -224,7 +214,27 @@ public class EdqsProcessor implements TbQueueHandler, }); } + private EdqsResponse processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + EdqsResponse response = new EdqsResponse(); + try { + if (request.getEntityDataQuery() != null) { + PageData result = repository.findEntityDataByQuery(tenantId, customerId, + request.getEntityDataQuery(), false); + response.setEntityDataQueryResult(result.mapData(QueryResult::toOldEntityData)); + } else if (request.getEntityCountQuery() != null) { + long result = repository.countEntitiesByQuery(tenantId, customerId, request.getEntityCountQuery(), tenantId.isSysTenantId()); + response.setEntityCountQueryResult(result); + } + log.trace("[{}] Request: {}, response: {}", tenantId, request, response); + } catch (Throwable e) { + log.error("[{}] Failed to process request: {}", tenantId, request, e); + response.setError(ExceptionUtil.getMessage(e)); + } + return response; + } + public void process(ToEdqsMsg edqsMsg, EdqsQueue queue) { + log.trace("Processing message: {}", edqsMsg); if (edqsMsg.hasEventMsg()) { EdqsEventMsg eventMsg = edqsMsg.getEventMsg(); TenantId tenantId = getTenantId(edqsMsg); @@ -290,7 +300,6 @@ public class EdqsProcessor implements TbQueueHandler, taskExecutor.shutdownNow(); scheduler.shutdownNow(); requestExecutor.shutdownNow(); - repartitionExecutor.shutdownNow(); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java index bfd0d9df59..9ea027ec10 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -27,6 +27,7 @@ import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; @@ -35,26 +36,27 @@ public class EdqsProducer { private final EdqsQueue queue; private final EdqsPartitionService partitionService; + private final TopicService topicService; private final TbQueueProducer> producer; @Builder public EdqsProducer(EdqsQueue queue, EdqsPartitionService partitionService, + TopicService topicService, TbQueueProducer> producer) { this.queue = queue; this.partitionService = partitionService; + this.topicService = topicService; this.producer = producer; } - // TODO: queue prefix! - public void send(TenantId tenantId, ObjectType type, String key, ToEdqsMsg msg) { - String topic = queue.getTopic(); + String topic = topicService.buildTopicName(queue.getTopic()); TbQueueCallback callback = new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); // fixme log levels + log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java similarity index 89% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java index 7736f303ac..bdfe76552a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java @@ -39,7 +39,7 @@ import java.util.function.Predicate; @AllArgsConstructor @Service @Slf4j -public class InMemoryEdqRepository implements EdqRepository { +public class DefaultEdqsRepository implements EdqsRepository { private final static ConcurrentMap repos = new ConcurrentHashMap<>(); private final Optional statsService; @@ -51,7 +51,7 @@ public class InMemoryEdqRepository implements EdqRepository { @Override public void processEvent(EdqsEvent event) { if (event.getEventType() == EdqsEventType.DELETED && event.getObjectType() == ObjectType.TENANT) { - log.info("Deleting tenant repo: {}", event); + log.info("Tenant {} deleted", event.getTenantId()); repos.remove(event.getTenantId()); } else { get(event.getTenantId()).processEvent(event); @@ -62,25 +62,25 @@ public class InMemoryEdqRepository implements EdqRepository { public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); long result = 0; - if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + if (!tenantId.isSysTenantId()) { + result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); + } else { for (TenantRepo repo : repos.values()) { result += repo.countEntitiesByQuery(customerId, query, ignorePermissionCheck); } - } else { - result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); } double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; - log.info("countEntitiesByQuery: {} ms", timingMs); + log.info("countEntitiesByQuery done in {} ms", timingMs); return result; } @Override public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, - EntityDataQuery query, boolean ignorePermissionCheck) { + EntityDataQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); var result = get(tenantId).findEntityDataByQuery(customerId, query, ignorePermissionCheck); double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; - log.info("findEntityDataByQuery: {} ms", timingMs); + log.info("findEntityDataByQuery done in {} ms", timingMs); return result; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java similarity index 92% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java index 231efd9a81..b1c72b1483 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java @@ -25,13 +25,10 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import java.util.function.Predicate; -public interface EdqRepository { +public interface EdqsRepository { void processEvent(EdqsEvent event); - @Deprecated - default void addOrUpdate(TenantId tenantId, Object object) {} - long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck); PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 9d6a4ab135..043c0bd6c3 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -57,6 +57,7 @@ import org.thingsboard.server.edqs.query.processor.EntityQueryProcessor; import org.thingsboard.server.edqs.query.processor.EntityQueryProcessorFactory; import org.thingsboard.server.edqs.stats.EdqsStatsService; import org.thingsboard.server.edqs.util.RepositoryUtils; +import org.thingsboard.server.edqs.util.TbStringPool; import java.util.ArrayList; import java.util.Collections; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index f7382c9f15..f5ba2ba841 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerMana import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueStateService; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.edqs.EdqsQueueFactory; @@ -52,6 +53,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsPartitionService partitionService; private final EdqsQueueFactory queueFactory; private final EdqsProcessor edqsProcessor; + private final TopicService topicService; private PartitionedQueueConsumerManager> stateConsumer; private QueueStateService, TbProtoQueueMsg> queueStateService; @@ -75,7 +77,6 @@ public class KafkaEdqsStateService implements EdqsStateService { } try { ToEdqsMsg msg = queueMsg.getValue(); - log.trace("Processing message: {}", msg); edqsProcessor.process(msg, EdqsQueue.STATE); if (stateReadCount.incrementAndGet() % 100000 == 0) { log.info("[state] Processed {} msgs", stateReadCount.get()); @@ -140,6 +141,7 @@ public class KafkaEdqsStateService implements EdqsStateService { stateProducer = EdqsProducer.builder() .queue(EdqsQueue.STATE) .partitionService(partitionService) + .topicService(topicService) .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.STATE)) .build(); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index b7419451b9..817b0df4f1 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -42,7 +42,6 @@ import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; import org.thingsboard.server.edqs.data.dp.JsonDataPoint; import org.thingsboard.server.edqs.data.dp.LongDataPoint; import org.thingsboard.server.edqs.data.dp.StringDataPoint; -import org.thingsboard.server.edqs.repo.TbBytePool; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; import org.xerial.snappy.Snappy; @@ -161,11 +160,11 @@ public class EdqsConverter { } else if (proto.hasStringV()) { return new StringDataPoint(ts, proto.getStringV()); } else if (proto.hasCompressedStringV()) { - return new CompressedStringDataPoint(ts, TbBytePool.intern(proto.getCompressedStringV().toByteArray())); + return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray()); } else if (proto.hasJsonV()) { return new JsonDataPoint(ts, proto.getJsonV()); } else if (proto.hasCompressedJsonV()) { - return new CompressedJsonDataPoint(ts, TbBytePool.intern(proto.getCompressedJsonV().toByteArray())); + return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray()); } else { throw new IllegalArgumentException("Unsupported data point proto: " + proto); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java similarity index 96% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java index a284590161..a5a5394910 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs.repo; +package org.thingsboard.server.edqs.util; import com.google.common.hash.Hashing; import org.springframework.util.ConcurrentReferenceHashMap; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java similarity index 96% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java index ceb15affdf..151d6bb1b4 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs.repo; +package org.thingsboard.server.edqs.util; import org.springframework.util.ConcurrentReferenceHashMap; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java index 4b599670a1..9f973ce3f1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -27,6 +27,7 @@ import org.thingsboard.server.queue.TbQueueResponseTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -47,12 +48,13 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { private final EdqsConfig edqsConfig; private final TbServiceInfoProvider serviceInfoProvider; private final TbKafkaConsumerStatsService consumerStatsService; + private final TopicService topicService; private final AtomicInteger consumerCounter = new AtomicInteger(); public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs, EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider, - TbKafkaConsumerStatsService consumerStatsService) { + TbKafkaConsumerStatsService consumerStatsService, TopicService topicService) { this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs()); this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsRequestsConfigs()); this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs()); @@ -60,6 +62,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { this.edqsConfig = edqsConfig; this.serviceInfoProvider = serviceInfoProvider; this.consumerStatsService = consumerStatsService; + this.topicService = topicService; } @Override @@ -72,11 +75,11 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) - .topic(queue.getTopic()) + .topic(topicService.buildTopicName(queue.getTopic())) .readFromBeginning(queue.isReadFromBeginning()) .stopWhenRead(queue.isStopWhenRead()) .clientId("edqs-" + queue.name().toLowerCase() + "-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId()) - .groupId(group) + .groupId(topicService.buildTopicName(group)) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) .statsService(consumerStatsService) @@ -97,16 +100,16 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { String requestsConsumerGroup = "edqs-requests-consumer-group-" + edqsConfig.getLabel(); var requestConsumer = TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) - .topic(edqsConfig.getRequestsTopic()) + .topic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) .clientId("edqs-requests-consumer-" + serviceInfoProvider.getServiceId()) - .groupId(requestsConsumerGroup) + .groupId(topicService.buildTopicName(requestsConsumerGroup)) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(edqsRequestsAdmin) .statsService(consumerStatsService); var responseProducer = TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId()) - .defaultTopic(edqsConfig.getResponsesTopic()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getResponsesTopic())) .admin(edqsRequestsAdmin); return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() .requestTemplate(requestConsumer.build()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index e5b981bf5d..4acb5d94ae 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -522,7 +522,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi var responseConsumer = TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) - .clientId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .clientId("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(edqsRequestsAdmin) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 4e12696d3e..646cdc676a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -471,7 +471,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { var responseConsumer = TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) - .clientId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .clientId("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(edqsRequestsAdmin) diff --git a/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java b/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java index ace7d1540b..21b20fec7b 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java @@ -64,4 +64,14 @@ public class ExceptionUtil { } } } + + public static String getMessage(Throwable t) { + String message = t.getMessage(); + if (StringUtils.isNotEmpty(message)) { + return message; + } else { + return t.getClass().getSimpleName(); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index 19fe45ed88..f6391fd26f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -30,8 +29,4 @@ public interface TenantEntityDao { throw new UnsupportedOperationException(); } - default ObjectType getType() { - throw new UnsupportedOperationException(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 24f8aa4d77..86887e94ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -18,9 +18,7 @@ package org.thingsboard.server.dao.entity; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.dao.Dao; -import org.thingsboard.server.dao.TenantEntityDao; import java.util.EnumMap; import java.util.List; @@ -31,20 +29,14 @@ import java.util.Map; @SuppressWarnings({"unchecked"}) public class EntityDaoRegistry { - private final Map> tenantEntityDaos = new EnumMap<>(ObjectType.class); private final Map> entityDaos = new EnumMap<>(EntityType.class); - private EntityDaoRegistry(List> entityDaos, List> tenantEntityDaos) { + private EntityDaoRegistry(List> entityDaos) { entityDaos.forEach(dao -> { if (dao.getEntityType() != null) { this.entityDaos.put(dao.getEntityType(), dao); } }); - tenantEntityDaos.forEach(dao -> { - if (dao.getType() != null) { - this.tenantEntityDaos.put(dao.getType(), dao); - } - }); } public Dao getDao(EntityType entityType) { @@ -55,12 +47,4 @@ public class EntityDaoRegistry { return dao; } - public TenantEntityDao getTenantEntityDao(ObjectType objectType) { - TenantEntityDao dao = tenantEntityDaos.get(objectType); - if (dao == null) { - throw new IllegalArgumentException("Missing tenant entity dao for entity type " + objectType); - } - return (TenantEntityDao) dao; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java deleted file mode 100644 index 729d7a44f0..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.model.sql; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.IdClass; -import jakarta.persistence.Table; -import lombok.Data; -import org.thingsboard.server.common.data.alarm.AlarmType; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.model.ToData; - -import java.util.UUID; - -@Data -@Entity -@Table(name = ModelConstants.ALARM_TYPES_TABLE_NAME) -@IdClass(AlarmTypeCompositeKey.class) -public class AlarmTypeEntity implements ToData { - - @Id - @Column(name = ModelConstants.TENANT_ID_PROPERTY, nullable = false) - private UUID tenantId; - - @Id - @Column(name = ModelConstants.ALARM_TYPE_PROPERTY, nullable = false) - private String type; - - public AlarmTypeEntity() {} - - public AlarmTypeEntity(AlarmType alarmType) { - setTenantId(alarmType.getTenantId().getId()); - setType(alarmType.getType()); - } - - public AlarmTypeEntity(UUID tenantId, String type) { - this.tenantId = tenantId; - this.type = type; - } - - @Override - public AlarmType toData() { - AlarmType alarmType = new AlarmType(); - alarmType.setTenantId(TenantId.fromUUID(tenantId)); - alarmType.setType(type); - return alarmType; - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java deleted file mode 100644 index b84542e3c3..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.alarm; - -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.dao.model.sql.AlarmTypeCompositeKey; -import org.thingsboard.server.dao.model.sql.AlarmTypeEntity; - -import java.util.UUID; - -public interface AlarmTypeRepository extends JpaRepository { - - Page findByTenantId(UUID tenantId, Pageable pageable); - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java index 9d74b9bf8e..f6d85fc762 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java @@ -22,7 +22,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmCommentInfo; import org.thingsboard.server.common.data.id.AlarmId; @@ -93,9 +92,4 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements A return EntityType.ALARM; } - @Override - public ObjectType getType() { - return ObjectType.ALARM; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java deleted file mode 100644 index 532f06eb03..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.alarm; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.ObjectType; -import org.thingsboard.server.common.data.alarm.AlarmType; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.TenantEntityDao; -import org.thingsboard.server.dao.util.SqlDao; - -@Component -@SqlDao -public class JpaAlarmTypeDao implements TenantEntityDao { - - @Autowired - private AlarmTypeRepository alarmTypeRepository; - - @Override - public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { - return DaoUtil.toPageData(alarmTypeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "tenantId", "type"))); - } - - @Override - public ObjectType getType() { - return ObjectType.ALARM_TYPE; - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java index 6bbc59559a..ebffd50656 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.alarm; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.alarm.EntityAlarm; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -38,9 +37,4 @@ public class JpaEntityAlarmDao implements TenantEntityDao { return DaoUtil.toPageData(entityAlarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "entityId", "alarmId"))); } - @Override - public ObjectType getType() { - return ObjectType.ENTITY_ALARM; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 176eb0856a..721f46901a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -23,7 +23,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.edqs.fields.AssetFields; @@ -285,9 +284,4 @@ public class JpaAssetDao extends JpaAbstractDao implements A return EntityType.ASSET; } - @Override - public ObjectType getType() { - return ObjectType.ASSET; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java index d43871823e..55f72d9682 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java @@ -22,7 +22,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; @@ -157,9 +156,4 @@ public class JpaAssetProfileDao extends JpaAbstractDao imp return EntityType.CUSTOMER; } - @Override - public ObjectType getType() { - return ObjectType.CUSTOMER; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 9422f38aae..1431837864 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -21,7 +21,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; @@ -108,9 +107,4 @@ public class JpaDashboardDao extends JpaAbstractDao return EntityType.DASHBOARD; } - @Override - public ObjectType getType() { - return ObjectType.DASHBOARD; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index 18f7f35710..4b87f0274d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; -import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 31d2aa6cb9..25b996eb57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -76,9 +75,4 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao implement return EntityType.DEVICE; } - @Override - public ObjectType getType() { - return ObjectType.DEVICE; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index c4d48dff35..8f7dcfb916 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -175,9 +174,4 @@ public class JpaDeviceProfileDao extends JpaAbstractDao implements Edge return EntityType.EDGE; } - @Override - public ObjectType getType() { - return ObjectType.EDGE; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index b18fd58248..3cc212497b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.edqs.fields.EntityViewFields; -import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 4c30e26f05..4172af056f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.EntityViewFields; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; @@ -236,9 +235,4 @@ public class JpaEntityViewDao extends JpaAbstractDao implements Q return EntityType.QUEUE; } - @Override - public ObjectType getType() { - return ObjectType.QUEUE; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index 41e2de02a6..3e8f2c11ce 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -21,7 +21,6 @@ import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; @@ -67,11 +66,6 @@ public class JpaQueueStatsDao extends JpaAbstractDao implements RpcDao, return EntityType.RPC; } - @Override - public ObjectType getType() { - return ObjectType.RPC; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index ad0d596474..d5cd046e98 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -21,7 +21,6 @@ import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -152,9 +151,4 @@ public class JpaRuleChainDao extends JpaAbstractDao return EntityType.RULE_CHAIN; } - @Override - public ObjectType getType() { - return ObjectType.RULE_CHAIN; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index 307f15c7f4..43087b8652 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -118,9 +117,4 @@ public class JpaRuleNodeDao extends JpaAbstractDao imp return EntityType.RULE_NODE; } - @Override - public ObjectType getType() { - return ObjectType.RULE_NODE; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 9e0f6a70cb..6a4ae62b48 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -21,7 +21,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -78,9 +77,4 @@ public class JpaAdminSettingsDao extends JpaAbstractDao implements User return EntityType.USER; } - @Override - public ObjectType getType() { - return ObjectType.USER; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index b41bfa7728..34f496e0a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sql.user; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; @@ -74,9 +73,4 @@ public class JpaUserSettingsDao implements UserSettingsDao, TenantEntityDao ignored = EnumSet.of(TENANT, TENANT_PROFILE, RELATION, EVENT, ATTRIBUTE_KV, LATEST_TS_KV, AUDIT_LOG, - OAUTH2_CLIENT, OAUTH2_DOMAIN, OAUTH2_MOBILE); - for (ObjectType type : ObjectType.values()) { - if (ignored.contains(type)) { - continue; - } - - TenantEntityDao dao = assertDoesNotThrow(() -> entityDaoRegistry.getTenantEntityDao(type)); - assertDoesNotThrow(() -> { - dao.findAllByTenantId(tenantId, new PageLink(100)); - }); - } - } - } diff --git a/edqs/src/main/resources/logback.xml b/edqs/src/main/resources/logback.xml index 8680457ab1..d96cb1a7d3 100644 --- a/edqs/src/main/resources/logback.xml +++ b/edqs/src/main/resources/logback.xml @@ -28,7 +28,6 @@ - diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java index 1f22f9d5ae..6f38e7b6d6 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -69,14 +69,14 @@ import java.util.UUID; @Configuration @ComponentScan({"org.thingsboard.server.edqs.repo", "org.thingsboard.server.edqs.util"}) @EntityScan("org.thingsboard.server.edqs") -@TestPropertySource(locations = {"classpath:edq-test.properties"}) +@TestPropertySource(locations = {"classpath:edqs-test.properties"}) @TestExecutionListeners({ DependencyInjectionTestExecutionListener.class, DirtiesContextTestExecutionListener.class}) public abstract class AbstractEDQTest { @Autowired - protected InMemoryEdqRepository repository; + protected DefaultEdqsRepository repository; @Autowired protected EdqsConverter edqsConverter; diff --git a/edqs/src/test/resources/edq-test.properties b/edqs/src/test/resources/edqs-test.properties similarity index 100% rename from edqs/src/test/resources/edq-test.properties rename to edqs/src/test/resources/edqs-test.properties From 965210f17b58b0ca42f698cf58178c247f7cca44 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 21 Feb 2025 14:39:26 +0200 Subject: [PATCH 23/51] Multiple improvements for EDQS --- .../src/main/resources/thingsboard.yml | 10 +- .../EdqsEntityQueryControllerTest.java | 8 +- .../server/edqs/processor/EdqsProcessor.java | 5 +- .../server/edqs/repo/TenantRepo.java | 2 +- .../server/edqs/state/EdqsStateService.java | 2 + .../edqs/state/KafkaEdqsStateService.java | 23 ++- .../edqs/state/LocalEdqsStateService.java | 5 + .../server/edqs/stats/EdqsStatsService.java | 7 +- .../common/msg/queue/TopicPartitionInfo.java | 11 +- .../consumer/MainQueueConsumerManager.java | 11 +- .../PartitionedQueueConsumerManager.java | 4 +- .../common/consumer/QueueStateService.java | 17 ++ .../queue/discovery/HashPartitionService.java | 2 +- .../server/queue/edqs/EdqsComponent.java | 1 - .../queue/kafka/TbKafkaConsumerTemplate.java | 4 +- .../common/stats/DefaultStatsFactory.java | 2 +- .../server/dao/entity/BaseEntityService.java | 4 +- .../dao/sql/query/DummyEdqsService.java | 4 +- .../server/edqs/EdqsController.java | 41 +++++ .../edqs/ThingsboardEdqsApplication.java | 67 +------- edqs/src/main/resources/edqs.yml | 147 ++---------------- edqs/src/main/resources/logback.xml | 1 + 22 files changed, 139 insertions(+), 239 deletions(-) create mode 100644 edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 21459814b7..46ab8f25de 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1696,14 +1696,14 @@ queue: print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" edqs: sync: - # Enable/disable EDQS synchronization FIXME: disable by default before release - enabled: "${TB_EDQS_SYNC_ENABLED:true}" + # Enable/disable EDQS synchronization + enabled: "${TB_EDQS_SYNC_ENABLED:false}" # Batch size of entities being synced with EDQS entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" # Batch size of timeseries data being synced with EDQS ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" - # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) FIXME: disable by default before release - api_enabled: "${TB_EDQS_API_ENABLED:true}" + # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) + api_enabled: "${TB_EDQS_API_ENABLED:false}" # Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices) mode: "${TB_EDQS_MODE:local}" local: @@ -1727,7 +1727,7 @@ queue: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" # Statistics printing interval for EDQS - print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" vc: # Default topic name diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index efc290d3c5..e4948330a5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.util.EdqsRocksDb; import java.util.concurrent.TimeUnit; @@ -34,7 +35,7 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { // "queue.type=kafka", // uncomment to use Kafka -// "queue.kafka.bootstrap.servers=192.168.0.105:9092", +// "queue.kafka.bootstrap.servers=10.7.1.254:9092", "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" @@ -44,12 +45,15 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { @Autowired private EdqsService edqsService; + @Autowired(required = false) + private EdqsStateService edqsStateService; + @MockBean // so that we don't do backup for tests private EdqsRocksDb edqsRocksDb; @Before public void before() { - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled() && edqsStateService.isReady()); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 42e00be7fc..9349946a80 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -24,9 +24,7 @@ import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ExceptionUtil; @@ -90,8 +88,7 @@ public class EdqsProcessor implements TbQueueHandler, private final EdqsConfig config; private final EdqsPartitionService partitionService; private final ConfigurableApplicationContext applicationContext; - @Autowired @Lazy - private EdqsStateService stateService; + private final EdqsStateService stateService; private PartitionedQueueConsumerManager> eventConsumer; private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 043c0bd6c3..9003eb4da2 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -105,7 +105,7 @@ public class TenantRepo { public void processEvent(EdqsEvent event) { EdqsObject edqsObject = event.getObject(); - log.debug("[{}] Processing event: {}", tenantId, event); + log.trace("[{}] Processing event: {}", tenantId, event); if (event.getEventType() == EdqsEventType.UPDATED) { addOrUpdate(edqsObject); } else if (event.getEventType() == EdqsEventType.DELETED) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java index 1966618d4b..bb0aa67239 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -33,6 +33,8 @@ public interface EdqsStateService { void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); + boolean isReady(); + void stop(); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index f5ba2ba841..05eebb188c 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.edqs.state; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; @@ -52,8 +54,9 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; private final EdqsQueueFactory queueFactory; - private final EdqsProcessor edqsProcessor; private final TopicService topicService; + @Autowired @Lazy + private EdqsProcessor edqsProcessor; private PartitionedQueueConsumerManager> stateConsumer; private QueueStateService, TbProtoQueueMsg> queueStateService; @@ -63,6 +66,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private final VersionsStore versionsStore = new VersionsStore(); private final AtomicInteger stateReadCount = new AtomicInteger(); private final AtomicInteger eventsReadCount = new AtomicInteger(); + private Boolean ready; @Override public void init(PartitionedQueueConsumerManager> eventConsumer) { @@ -72,9 +76,6 @@ public class KafkaEdqsStateService implements EdqsStateService { .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { - if (consumer.isStopped()) { - return; - } try { ToEdqsMsg msg = queueMsg.getValue(); edqsProcessor.process(msg, EdqsQueue.STATE); @@ -124,7 +125,7 @@ public class KafkaEdqsStateService implements EdqsStateService { TenantId tenantId = getTenantId(msg); ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); - log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + log.trace("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); stateProducer.send(tenantId, objectType, key, msg); } } catch (Throwable t) { @@ -160,6 +161,18 @@ public class KafkaEdqsStateService implements EdqsStateService { // do nothing here, backup is done by events consumer } + @Override + public boolean isReady() { + if (ready == null) { + Set partitionsInProgress = queueStateService.getPartitionsInProgress(); + if (partitionsInProgress != null && partitionsInProgress.isEmpty()) { + ready = true; // once true - always true, not to change readiness status on each repartitioning + } + } + log.error("ready: {}", ready); + return ready != null && ready; + } + private TenantId getTenantId(ToEdqsMsg edqsMsg) { return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index 6869f14b74..c1620e5ec7 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -80,6 +80,11 @@ public class LocalEdqsStateService implements EdqsStateService { } } + @Override + public boolean isReady() { + return partitions != null; + } + @Override public void stop() { } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java index 9619d086ad..d28334c5d9 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -43,9 +43,12 @@ public class EdqsStatsService { private final ConcurrentHashMap statsMap = new ConcurrentHashMap<>(); private final StatsFactory statsFactory; - @Scheduled(initialDelayString = "${queue.edqs.stats.print-interval-ms:60000}", - fixedDelayString = "${queue.edqs.stats.print-interval-ms:60000}") + @Scheduled(initialDelayString = "${queue.edqs.stats.print-interval-ms:300000}", + fixedDelayString = "${queue.edqs.stats.print-interval-ms:300000}") private void reportStats() { + if (statsMap.isEmpty()) { + return; + } String values = statsMap.entrySet().stream() .map(kv -> "TenantId [" + kv.getKey() + "] stats [" + kv.getValue() + "]") .collect(Collectors.joining(System.lineSeparator())); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index 3bb489f209..da7f16ec0b 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -17,13 +17,11 @@ package org.thingsboard.server.common.msg.queue; import lombok.Builder; import lombok.Getter; -import lombok.ToString; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; import java.util.Optional; -@ToString public class TopicPartitionInfo { private final String topic; @@ -97,4 +95,13 @@ public class TopicPartitionInfo { return Objects.hash(fullTopicName, partition); } + @Override + public String toString() { + String str = fullTopicName; + if (useInternalPartition) { + str += "[" + partition + "]"; + } + return str; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index df683b94cd..a73ee17bd2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -43,7 +43,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; import java.util.function.Consumer; -import java.util.stream.Collectors; @Slf4j public class MainQueueConsumerManager { @@ -270,12 +269,6 @@ public class MainQueueConsumerManager partitions) { - return partitions.stream().map(tpi -> tpi.getFullTopicName() + (tpi.isUseInternalPartition() ? - "[" + tpi.getPartition().orElse(-1) + "]" : "")) - .collect(Collectors.joining(", ", "[", "]")); - } - public interface MsgPackProcessor { void process(List msgs, TbQueueConsumer consumer, C config) throws Exception; } @@ -299,7 +292,7 @@ public class MainQueueConsumerManager removedPartitions = new HashSet<>(consumers.keySet()); removedPartitions.removeAll(partitions); - log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions)); + log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, addedPartitions, removedPartitions); removePartitions(removedPartitions); addPartitions(addedPartitions, null); } @@ -333,7 +326,7 @@ public class MainQueueConsumerManager partitions) { - log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions)); + log.info("[{}] New partitions: {}", queueKey, partitions); if (partitions.isEmpty()) { if (consumer != null && consumer.isRunning()) { consumer.initiateStop(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java index 1b47d4c073..6ab0668f30 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -52,10 +52,10 @@ public class PartitionedQueueConsumerManager extends MainQ @Override protected void processTask(TbQueueConsumerManagerTask task) { if (task instanceof AddPartitionsTask addPartitionsTask) { - log.info("[{}] Added partitions: {}", queueKey, partitionsToString(addPartitionsTask.partitions())); + log.info("[{}] Added partitions: {}", queueKey, addPartitionsTask.partitions()); consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop()); } else if (task instanceof RemovePartitionsTask removePartitionsTask) { - log.info("[{}] Removed partitions: {}", queueKey, partitionsToString(removePartitionsTask.partitions())); + log.info("[{}] Removed partitions: {}", queueKey, removePartitionsTask.partitions()); consumerWrapper.removePartitions(removePartitionsTask.partitions()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java index d022ad14de..00015457c1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java @@ -16,16 +16,19 @@ package org.thingsboard.server.queue.common.consumer; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueMsg; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +@Slf4j public class QueueStateService { private PartitionedQueueConsumerManager stateConsumer; @@ -33,6 +36,9 @@ public class QueueStateService { @Getter private Set partitions; + private final Set partitionsInProgress = ConcurrentHashMap.newKeySet(); + private boolean initialized; + private final Lock lock = new ReentrantLock(); public void init(PartitionedQueueConsumerManager stateConsumer, PartitionedQueueConsumerManager eventConsumer) { @@ -62,9 +68,15 @@ public class QueueStateService { } if (!addedPartitions.isEmpty()) { + partitionsInProgress.addAll(addedPartitions); stateConsumer.addPartitions(addedPartitions, partition -> { lock.lock(); try { + partitionsInProgress.remove(partition); + log.info("Finished partition {} (still in progress: {})", partition, partitionsInProgress); + if (partitionsInProgress.isEmpty()) { + log.info("All partitions processed"); + } if (this.partitions.contains(partition)) { eventConsumer.addPartitions(Set.of(partition.withTopic(eventConsumer.getTopic()))); } @@ -73,10 +85,15 @@ public class QueueStateService { } }); } + initialized = true; } private Set withTopic(Set partitions, String topic) { return partitions.stream().map(tpi -> tpi.withTopic(topic)).collect(Collectors.toSet()); } + public Set getPartitionsInProgress() { + return initialized ? partitionsInProgress : null; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index ea9ba3f245..5787516b25 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -59,7 +59,7 @@ import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; @Slf4j public class HashPartitionService implements PartitionService { - @Value("${queue.core.topic}") + @Value("${queue.core.topic:tb_core}") private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java index 27a1e09f79..ba113495d1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -21,7 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -// TODO: tb-core ? @ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + "'${queue.edqs.mode:null}'=='local'))") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 47c776c742..9b580c762e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -54,6 +54,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final boolean readFromBeginning; // reset offset to beginning private final boolean stopWhenRead; // stop consuming when reached an empty msg pack + private int readCount; private Map endOffsets; // needed if stopWhenRead is true private boolean partitionsAssigned = false; @@ -152,6 +153,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue records.forEach(record -> { recordList.add(record); if (stopWhenRead) { + readCount++; int partition = record.partition(); Long endOffset = endOffsets.get(partition); if (endOffset == null) { @@ -166,7 +168,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue }); } if (stopWhenRead && endOffsets.isEmpty()) { - log.info("Reached end offset for {}, stopping consumer", consumer.assignment()); + log.info("Finished reading {}, processed {} messages", partitions, readCount); stop(); } return recordList; diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index 2fc7970fc9..e1940e55fe 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -38,7 +38,7 @@ public class DefaultStatsFactory implements StatsFactory { private static final Counter STUB_COUNTER = new StubCounter(); - @Autowired(required = false) // FIXME Slavik !!! + @Autowired // FIXME Slavik !!! private MeterRegistry meterRegistry; @Value("${metrics.enabled:false}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3f941b776c..1a0f3be1e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -142,12 +142,12 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe private EdqsResponse processEdqsRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { EdqsResponse response; try { - log.info("Sending request to EDQS: {}", request); + log.debug("[{}] Sending request to EDQS: {}", tenantId, request); response = edqsService.processRequest(tenantId, customerId, request).get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } - log.info("Received response from EDQS: {}", response); + log.debug("[{}] Received response from EDQS: {}", tenantId, response); if (response.getError() != null) { throw new RuntimeException(response.getError()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java index b69aa52ec5..9586cb62f8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -16,7 +16,7 @@ package org.thingsboard.server.dao.sql.query; import com.google.common.util.concurrent.ListenableFuture; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsObject; @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.edqs.EdqsService; @Service -@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "false", matchIfMissing = true) +@ConditionalOnMissingBean(value = EdqsService.class, ignored = DummyEdqsService.class) public class DummyEdqsService implements EdqsService { @Override diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java b/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java new file mode 100644 index 0000000000..f2e686a84c --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.edqs.state.EdqsStateService; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/edqs") +public class EdqsController { + + private final EdqsStateService edqsStateService; + + @GetMapping("/ready") + public ResponseEntity isReady() { + if (edqsStateService.isReady()) { + return ResponseEntity.ok().build(); + } else { + return ResponseEntity.badRequest().build(); + } + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java index 044020023d..2f5185728a 100644 --- a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java +++ b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java @@ -18,6 +18,7 @@ package org.thingsboard.server.edqs; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @@ -27,6 +28,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling +@EnableAutoConfiguration @ComponentScan({"org.thingsboard.server.edqs", "org.thingsboard.server.queue.edqs", "org.thingsboard.server.queue.discovery", "org.thingsboard.server.queue.kafka", "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment", "org.thingsboard.server.common.stats"}) @Slf4j @@ -39,71 +41,6 @@ public class ThingsboardEdqsApplication { SpringApplication.run(ThingsboardEdqsApplication.class, updateArguments(args)); } - // @Bean -// public ApplicationRunner runner(CSVLoader loader, EdqRepository edqRepository) { -// return args -> { -// long startTs = System.currentTimeMillis(); -// var loader = new TenantRepoLoader(new TenantRepo(TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")))); -// loader.load(); -// log.info("Loaded all in {} ms", System.currentTimeMillis() - startTs); - - - -// log.info("Compressed {} strings/json, Before: {}, After: {}", -// CompressedStringDataPoint.cnt.get(), -// CompressedStringDataPoint.uncompressedLength.get(), -// CompressedStringDataPoint.compressedLength.get()); -// -// log.info("Deduplicated {} short and {} long strings", -// TbStringPool.size(), TbBytePool.size()); -// -// var tenantId = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); -// var customerId = new CustomerId(UUID.fromString("fcbf2f50-d0d9-11ea-bea3-177755191a6e")); -// System.gc(); -// -// while (true) { -// EntityTypeFilter filter = new EntityTypeFilter(); -// filter.setEntityType(EntityType.DEVICE); -// var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); -// -// var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); -// var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); -// KeyFilter nameFilter = new KeyFilter(); -// nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); -// var predicate = new StringFilterPredicate(); -// predicate.setIgnoreCase(false); -// predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); -// predicate.setValue(new FilterPredicateValue<>("LoRa-")); -// nameFilter.setPredicate(predicate); -// nameFilter.setValueType(EntityKeyValueType.STRING); -// -// EntityDataQuery edq = new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); -// var result = edqRepository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, edq, false); -// log.info("Device count: {}", result.getTotalElements()); -// log.info("First: {}", result.getData().get(0).getEntityId()); -// log.info("Last: {}", result.getData().get(19).getEntityId()); -// -// pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.ASC)); -// result = edqRepository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, edq, false); -// log.info("Device count: {}", result.getTotalElements()); -// log.info("First: {}", result.getData().get(0).getEntityId()); -// log.info("Last: {}", result.getData().get(19).getEntityId()); -// -// result.getData().forEach(data -> { -// System.err.println(data.getEntityId() + ":"); -// data.getLatest().forEach((type, values) -> { -// System.err.println(type); -// values.forEach((key, tsValue) -> { -// System.err.println(key + " = " + tsValue.getValue()); -// }); -// }); -// System.err.println(); -// }); -// Thread.sleep(5000); -// } -// }; -// } - private static String[] updateArguments(String[] args) { if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { String[] modifiedArgs = new String[args.length + 1]; diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index fa09da1cdd..3d30c5c6fa 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -14,6 +14,12 @@ # limitations under the License. # +server: + # Server bind-address + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port + port: "${HTTP_BIND_PORT:8080}" + # Application info parameters app: # Application version @@ -38,10 +44,9 @@ zk: # The delay is recommended because the initialization of rule chain actors is time-consuming. Avoiding unnecessary recalculations during a restart can enhance system performance and stability. recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" -# SQL DAO Configuration parameters spring: main: - web-application-type: "none" + allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Queue configuration parameters queue: @@ -66,7 +71,7 @@ queue: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" # Statistics printing interval for EDQS - print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" kafka: # Kafka Bootstrap nodes in "host:port" format @@ -137,9 +142,9 @@ queue: - key: max.poll.interval.ms # Example of specific consumer properties value per topic for VC value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" - # tb_rule_engine.sq: - # - key: max.poll.records - # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" tb_housekeeper: # Consumer properties for Housekeeper tasks topic - key: max.poll.records @@ -209,133 +214,6 @@ queue: request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" - core: - # Default topic name of Kafka, RabbitMQ, etc. queue - topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" - # Interval in milliseconds to poll messages by Core microservices - poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" - # Amount of partitions used by Core microservices - partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - # Timeout for processing a message pack by Core microservices - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" - # Enable/disable a separate consumer per partition for Core queue - consumer-per-partition: "${TB_QUEUE_CORE_CONSUMER_PER_PARTITION:true}" - ota: - # Default topic name for OTA updates - topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" - # The interval of processing the OTA updates for devices. Used to avoid any harm to the network due to many parallel OTA updates - pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" - # The size of OTA updates notifications fetched from the queue. The queue stores pairs of firmware and device ids - pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" - # Stats topic name for queue Kafka, RabbitMQ, etc. - usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" - stats: - # Enable/disable statistics for Core microservices - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" - # Statistics printing interval for Core microservices - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" - housekeeper: - # Topic name for Housekeeper tasks - topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" - # Topic name for Housekeeper tasks to be reprocessed - reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" - # Poll interval for topics related to Housekeeper - poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" - # Timeout in milliseconds for task processing. Tasks that fail to finish on time will be submitted for reprocessing - task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" - # Comma-separated list of task types that shouldn't be processed. Available task types: - # DELETE_ATTRIBUTES, DELETE_TELEMETRY (both DELETE_LATEST_TS and DELETE_TS_HISTORY will be disabled), - # DELETE_LATEST_TS, DELETE_TS_HISTORY, DELETE_EVENTS, DELETE_ALARMS, UNASSIGN_ALARMS - disabled-task-types: "${TB_HOUSEKEEPER_DISABLED_TASK_TYPES:}" - # Delay in milliseconds between tasks reprocessing - task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:3000}" - # Maximum amount of task reprocessing attempts. After exceeding, the task will be dropped - max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}" - stats: - # Enable/disable statistics for Housekeeper - enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" - # Statistics printing interval for Housekeeper - print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" - - vc: - # Default topic name for Kafka, RabbitMQ, etc. - topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" - # Number of partitions to associate with this queue. Used for scaling the number of messages that can be processed in parallel - partitions: "${TB_QUEUE_VC_PARTITIONS:10}" - # Interval in milliseconds between polling of the messages if no new messages arrive - poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" - # Timeout before retrying all failed and timed-out messages from the processing pack - pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" - # Timeout for a request to VC-executor (for a request for the version of the entity, for a commit charge, etc.) - request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:180000}" - # Queue settings for Kafka, RabbitMQ, etc. Limit for single message size - msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" - js: - # JS Eval request topic - request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" - # JS Eval responses topic prefix that is combined with node id - response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" - # JS Eval max pending requests - max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" - # JS Eval max request timeout - max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}" - # JS max request timeout - max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" - # JS execution max request timeout - max_exec_requests_timeout: "${REMOTE_JS_MAX_EXEC_REQUEST_TIMEOUT:2000}" - # JS response poll interval - response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" - rule-engine: - # Deprecated. It will be removed in the nearest releases - topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" - # Interval in milliseconds to poll messages by Rule Engine - poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - # Timeout for processing a message pack of Rule Engine - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}" - stats: - # Enable/disable statistics for Rule Engine - enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" - # Statistics printing interval for Rule Engine - print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" - # Max length of the error message that is printed by statistics - max-error-message-length: "${TB_QUEUE_RULE_ENGINE_MAX_ERROR_MESSAGE_LENGTH:4096}" - # After a queue is deleted (or the profile's isolation option was disabled), Rule Engine will continue reading related topics during this period before deleting the actual topics - topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" - # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion - management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" - transport: - # For high-priority notifications that require minimum latency and processing time - notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" - # Interval in milliseconds to poll messages - poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" - integration: - # Name of hash function used for consistent hash ring in Cluster Mode. See architecture docs for more details. Valid values - murmur3_32, murmur3_128 or sha256 - partitions: "${TB_QUEUE_INTEGRATION_PARTITIONS:3}" - # Default notification topic name used by queue - notifications_topic: "${TB_QUEUE_INTEGRATION_NOTIFICATIONS_TOPIC:tb_ie.notifications}" - # Default downlink topic name used by queue - downlink_topic: "${TB_QUEUE_INTEGRATION_DOWNLINK_TOPIC:tb_ie.downlink}" - # Default uplink topic name used by queue - uplink_topic: "${TB_QUEUE_INTEGRATION_UPLINK_TOPIC:tb_ie.uplink}" - # Interval in milliseconds to poll messages by integrations - poll_interval: "${TB_QUEUE_INTEGRATION_POLL_INTERVAL_MS:25}" - # Timeout for processing a message pack by integrations - pack-processing-timeout: "${TB_QUEUE_INTEGRATION_PACK_PROCESSING_TIMEOUT_MS:10000}" - integration_api: - # Default Integration Api request topic name used by queue - requests_topic: "${TB_QUEUE_INTEGRATION_EXECUTOR_API_REQUEST_TOPIC:tb_ie.api.requests}" - # Default Integration Api response topic name used by queue - responses_topic: "${TB_QUEUE_INTEGRATION_EXECUTOR_API_RESPONSE_TOPIC:tb_ie.api.responses}" - # Maximum pending api requests from integration executor to be handled by server< - max_pending_requests: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_PENDING_REQUESTS:10000}" - # Maximum timeout in milliseconds to handle api request from integration executor microservice by server - max_requests_timeout: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_REQUEST_TIMEOUT:10000}" - # Amount of threads used to invoke callbacks - max_callback_threads: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_CALLBACK_THREADS:10}" - # Interval in milliseconds to poll api requests from integration executor microservices - request_poll_interval: "${TB_QUEUE_INTEGRATION_EXECUTOR_REQUEST_POLL_INTERVAL_MS:25}" - # Interval in milliseconds to poll api response from integration executor microservices - response_poll_interval: "${TB_QUEUE_INTEGRATION_EXECUTOR_RESPONSE_POLL_INTERVAL_MS:25}" # General service parameters service: @@ -343,7 +221,8 @@ service: # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" edqs: - label: "${TB_EDQS_LABEL:}" # services with the same label will share the list of partitions + # EDQS instances with the same label will share the same list of partitions + label: "${TB_EDQS_LABEL:}" # Metrics parameters metrics: diff --git a/edqs/src/main/resources/logback.xml b/edqs/src/main/resources/logback.xml index d96cb1a7d3..0aedb9d092 100644 --- a/edqs/src/main/resources/logback.xml +++ b/edqs/src/main/resources/logback.xml @@ -26,6 +26,7 @@ + From 278ec36f376cd707562ec2b6b650f6eaf273832e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 21 Feb 2025 15:48:33 +0200 Subject: [PATCH 24/51] Fixes for EDQS --- .../server/edqs/processor/EdqsProcessor.java | 8 ++------ .../server/edqs/state/LocalEdqsStateService.java | 10 ++++++++-- .../server/common/msg/queue/TopicPartitionInfo.java | 6 ++++++ .../queue/common/AbstractTbQueueConsumerTemplate.java | 10 ++++------ .../queue/common/consumer/QueueStateService.java | 7 ++----- .../server/queue/edqs/InMemoryEdqsQueueFactory.java | 6 ++++-- .../server/queue/edqs/KafkaEdqsQueueFactory.java | 10 +++++++--- .../server/common/stats/DefaultStatsFactory.java | 2 +- 8 files changed, 34 insertions(+), 25 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 9349946a80..ac7af4691d 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -76,6 +76,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + @EdqsComponent @Service @RequiredArgsConstructor @@ -280,12 +282,6 @@ public class EdqsProcessor implements TbQueueHandler, } } - private Set withTopic(Set partitions, String topic) { - return partitions.stream() - .map(tpi -> tpi.withTopic(topic)) - .collect(Collectors.toSet()); - } - @PreDestroy public void destroy() throws InterruptedException { eventConsumer.stop(); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index c1620e5ec7..67724dd8b1 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.edqs.state; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; @@ -32,14 +34,17 @@ import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; import java.util.Set; +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + @Service @RequiredArgsConstructor @InMemoryEdqsComponent @Slf4j public class LocalEdqsStateService implements EdqsStateService { - private final EdqsProcessor processor; private final EdqsRocksDb db; + @Autowired @Lazy + private EdqsProcessor processor; private PartitionedQueueConsumerManager> eventConsumer; private Set partitions; @@ -61,8 +66,9 @@ public class LocalEdqsStateService implements EdqsStateService { log.error("[{}] Failed to restore value", key, e); } }); + log.info("Restore completed"); } - eventConsumer.update(partitions); + eventConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic())); this.partitions = partitions; } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index da7f16ec0b..c7a23451d1 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; public class TopicPartitionInfo { @@ -75,6 +77,10 @@ public class TopicPartitionInfo { return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); } + public static Set withTopic(Set partitions, String topic) { + return partitions.stream().map(tpi -> tpi.withTopic(topic)).collect(Collectors.toSet()); + } + public TopicPartitionInfo withUseInternalPartition(boolean useInternalPartition) { return new TopicPartitionInfo(this.topic, this.tenantId, this.partition, useInternalPartition, this.myPartition); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 0e399929f4..88e8f60dee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -31,7 +31,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -95,7 +94,7 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - log.info("Subscribing to topics {}", getFullTopicNames()); + log.info("Subscribing to {}", partitions); doSubscribe(partitions); subscribed = true; } @@ -168,7 +167,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { - log.info("Unsubscribing and stopping consumer for topics {}", getFullTopicNames()); + log.info("Unsubscribing and stopping consumer for {}", partitions); stopped = true; consumerLock.lock(); try { @@ -201,9 +200,8 @@ public abstract class AbstractTbQueueConsumerTemplate i return Collections.emptyList(); } return partitions.stream() - .map(tpi -> tpi.getFullTopicName() + (tpi.isUseInternalPartition() ? - "[" + tpi.getPartition().orElse(-1) + "]" : "")) - .collect(Collectors.toList()); + .map(TopicPartitionInfo::getFullTopicName) + .toList(); } protected boolean isLongPollingSupported() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java index 00015457c1..2671f03b95 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java @@ -26,7 +26,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; + +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; @Slf4j public class QueueStateService { @@ -88,10 +89,6 @@ public class QueueStateService { initialized = true; } - private Set withTopic(Set partitions, String topic) { - return partitions.stream().map(tpi -> tpi.withTopic(topic)).collect(Collectors.toSet()); - } - public Set getPartitionsInProgress() { return initialized ? partitionsInProgress : null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java index 20b4beadcf..dc6e9f4791 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java @@ -18,7 +18,8 @@ package org.thingsboard.server.queue.edqs; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.stats.DummyMessagesStats; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -37,6 +38,7 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { private final InMemoryStorage storage; private final EdqsConfig edqsConfig; + private final StatsFactory statsFactory; @Override public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { @@ -69,7 +71,7 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { .maxPendingRequests(edqsConfig.getMaxPendingRequests()) .requestTimeout(edqsConfig.getMaxRequestTimeout()) .pollInterval(edqsConfig.getPollInterval()) - .stats(new DummyMessagesStats()) // FIXME + .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) .build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java index 9f973ce3f1..9561849dd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -17,7 +17,8 @@ package org.thingsboard.server.queue.edqs; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.stats.DummyMessagesStats; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; @@ -49,12 +50,14 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { private final TbServiceInfoProvider serviceInfoProvider; private final TbKafkaConsumerStatsService consumerStatsService; private final TopicService topicService; + private final StatsFactory statsFactory; private final AtomicInteger consumerCounter = new AtomicInteger(); public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs, EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider, - TbKafkaConsumerStatsService consumerStatsService, TopicService topicService) { + TbKafkaConsumerStatsService consumerStatsService, TopicService topicService, + StatsFactory statsFactory) { this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs()); this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsRequestsConfigs()); this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs()); @@ -63,6 +66,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { this.serviceInfoProvider = serviceInfoProvider; this.consumerStatsService = consumerStatsService; this.topicService = topicService; + this.statsFactory = statsFactory; } @Override @@ -117,7 +121,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { .maxPendingRequests(edqsConfig.getMaxPendingRequests()) .requestTimeout(edqsConfig.getMaxRequestTimeout()) .pollInterval(edqsConfig.getPollInterval()) - .stats(new DummyMessagesStats()) // FIXME + .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) .build(); } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index e1940e55fe..ca97792186 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -38,7 +38,7 @@ public class DefaultStatsFactory implements StatsFactory { private static final Counter STUB_COUNTER = new StubCounter(); - @Autowired // FIXME Slavik !!! + @Autowired private MeterRegistry meterRegistry; @Value("${metrics.enabled:false}") From fb161730fb28fd4d53907426dd2038e5401f4b48 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 21 Feb 2025 18:38:26 +0200 Subject: [PATCH 25/51] added msa tests for edqs --- .../controller/EntityQueryController.java | 7 + .../entitiy/EdqsEntityServiceTest.java | 49 +++++ .../service/entitiy/EntityServiceTest.java | 3 +- .../common/data/edqs/fields/FieldsUtil.java | 4 +- .../edqs/repo/InMemoryEdqRepository.java | 9 +- .../server/dao/entity/BaseEntityService.java | 4 +- .../dao/sql/query/DummyEdqsService.java | 1 + docker/.env | 1 + docker/compose-utils.sh | 12 ++ docker/docker-compose.edqs.yml | 33 +++ docker/docker-remove-services.sh | 4 +- docker/docker-start-services.sh | 4 +- docker/docker-stop-services.sh | 4 +- docker/edqs.env | 7 + docker/tb-node.env | 2 + edqs/src/main/resources/edqs.yml | 5 + .../server/msa/ContainerTestSuite.java | 11 +- .../server/msa/TestRestClient.java | 100 ++++++++- .../server/msa/ThingsBoardDbInstaller.java | 3 +- .../msa/edqs/EdqsEntityDataQueryTest.java | 202 ++++++++++++++++++ .../server/msa/ui/utils/EntityPrototypes.java | 32 +++ 21 files changed, 477 insertions(+), 20 deletions(-) create mode 100644 docker/docker-compose.edqs.yml create mode 100644 docker/edqs.env create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index f28d485b30..9c2c70d019 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -20,6 +20,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -144,4 +145,10 @@ public class EntityQueryController extends BaseController { edqsService.processSystemRequest(request); } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @GetMapping("/edqs/enabled") + public boolean isEdqsApiEnabled() { + return edqsService.isApiEnabled(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index cb377e0431..61f246ca3e 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -15,20 +15,35 @@ */ package org.thingsboard.server.service.entitiy; +import com.google.common.collect.Lists; import org.junit.Before; +import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.awaitility.Awaitility.await; @@ -51,6 +66,40 @@ public class EdqsEntityServiceTest extends EntityServiceTest { await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); } + // sql implementation has a bug with data duplication, edqs implementation returns correct value + @Override + @Test + public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { + List buildings = new ArrayList<>(); + List apartments = new ArrayList<>(); + Map> entityNameByTypeMap = new HashMap<>(); + Map childParentRelationMap = new HashMap<>(); + createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap); + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setMultiRoot(true); + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(buildings.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.FROM); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + countByQueryAndCheck(countQuery, 63); + + filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 27); + + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.TO); + filter.setFilters(Lists.newArrayList( + new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), + new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 3); + + deviceService.deleteDevicesByTenantId(tenantId); + assetService.deleteAssetsByTenantId(tenantId); + } + @Override protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index a559eca6ce..44e636b04f 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -462,7 +462,6 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } - // fails for sql implementation until we fix the issue with the relation query @Test public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { List buildings = new ArrayList<>(); @@ -489,7 +488,7 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - countByQueryAndCheck(countQuery, 3); + countByQueryAndCheck(countQuery, 9); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index 6543a6d779..8d939e07e5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -240,7 +240,7 @@ public class FieldsUtil { .build(); } - private static AssetProfileFields toFields(DeviceProfile entity) { + private static AssetProfileFields toFields(AssetProfile entity) { return AssetProfileFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) @@ -250,7 +250,7 @@ public class FieldsUtil { .build(); } - private static DeviceProfileFields toFields(AssetProfile entity) { + private static DeviceProfileFields toFields(DeviceProfile entity) { return DeviceProfileFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java index 7736f303ac..1b6903fc3c 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java @@ -61,14 +61,7 @@ public class InMemoryEdqRepository implements EdqRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); - long result = 0; - if (TenantId.SYS_TENANT_ID.equals(tenantId)) { - for (TenantRepo repo : repos.values()) { - result += repo.countEntitiesByQuery(customerId, query, ignorePermissionCheck); - } - } else { - result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); - } + long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; log.info("countEntitiesByQuery: {} ms", timingMs); return result; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3f941b776c..7705614a05 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -95,7 +95,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); - if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always EdqsRequest request = EdqsRequest.builder() .entityCountQuery(query) .build(); @@ -112,7 +112,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); - if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always EdqsRequest request = EdqsRequest.builder() .entityDataQuery(query) .build(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java index b69aa52ec5..14ab995a45 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.query; import com.google.common.util.concurrent.ListenableFuture; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; diff --git a/docker/.env b/docker/.env index 37c9768296..38724263fc 100644 --- a/docker/.env +++ b/docker/.env @@ -14,6 +14,7 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor +EDQS_DOCKER_NAME=edqs TB_VERSION=latest diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index aa5a9db08d..9411c22b7e 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -128,6 +128,18 @@ function additionalStartupServices() { echo $ADDITIONAL_STARTUP_SERVICES } +function additionalComposeEdqsArgs() { + source .env + + if [ "$EDQS_ENABLED" = true ] + then + ADDITIONAL_COMPOSE_EDQS_ARGS="-f docker-compose.edqs.yml" + echo ADDITIONAL_COMPOSE_EDQS_ARGS + else + echo "" + fi +} + function permissionList() { PERMISSION_LIST=" 799 799 tb-node/log diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml new file mode 100644 index 0000000000..f989c12a2e --- /dev/null +++ b/docker/docker-compose.edqs.yml @@ -0,0 +1,33 @@ +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + edqs-1: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + depends_on: + - zookeeper + edqs-2: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + depends_on: + - zookeeper diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index c77499f833..d7fd891925 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ down -v" case $COMPOSE_VERSION in diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 42af042108..3d4479c001 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -29,10 +29,12 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + checkFolders --create || exit $? COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d" case $COMPOSE_VERSION in diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 2a753cc348..b7037fce69 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS}\ stop" case $COMPOSE_VERSION in diff --git a/docker/edqs.env b/docker/edqs.env new file mode 100644 index 0000000000..3c91c92e54 --- /dev/null +++ b/docker/edqs.env @@ -0,0 +1,7 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 +TB_KAFKA_SERVERS=kafka:9092 + +TB_EDQS_STATS_ENABLED=false +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus diff --git a/docker/tb-node.env b/docker/tb-node.env index 85a60eb51a..d40ca66621 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -9,3 +9,5 @@ HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus + +TB_EDQS_MODE=remote \ No newline at end of file diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index b67f2f5f9e..a55391451e 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -60,6 +60,11 @@ queue: poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + stats: + # Enable/disable statistics for EDQS service + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" kafka: # Kafka Bootstrap nodes in "host:port" format bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index bbaeabe907..d22861e286 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -114,6 +114,7 @@ public class ContainerTestSuite { List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.edqs.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.mosquitto.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), @@ -162,6 +163,12 @@ public class ContainerTestSuite { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); } + // to trigger edqs synchronization + if (true) { + addToFile(targetDir, "tb-node.env", + Map.of("TB_EDQS_SYNC_ENABLED", "true")); + } + testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) .withLocalCompose(true) @@ -180,7 +187,9 @@ public class ContainerTestSuite { .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("edqs-1", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("edqs-2", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index d137f8ed59..3c61c65ef1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -28,6 +28,7 @@ import io.restassured.internal.ValidatableResponseImpl; import io.restassured.path.json.JsonPath; import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; @@ -35,6 +36,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; @@ -56,6 +58,9 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rpc.Rpc; @@ -66,7 +71,6 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import static io.restassured.RestAssured.given; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; @@ -110,6 +114,37 @@ public class TestRestClient { requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); } + public JsonNode getActivateRequest(String password) { + ObjectNode response = given().spec(requestSpec).get("/api/noauth/activate?activateToken={activateToken}", token) + .then() + .statusCode(HTTP_OK) + .extract() + .as(ObjectNode.class); + return JacksonUtil.newObjectNode() + .put("activateToken", this.token) + .put("password", password); + } + + public void activateAndLoginAsUser(JsonNode activateRequest) { + ObjectNode tokenInfo = given().spec(requestSpec).body(activateRequest) + .post("/api/noauth/activate") + .then() + .extract() + .as(ObjectNode.class); + token = tokenInfo.get("token").asText(); + refreshToken = tokenInfo.get("refreshToken").asText(); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + public Tenant postTenant(Tenant tenant) { + return given().spec(requestSpec).body(tenant) + .post("/api/tenant") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Tenant.class); + } + public Device postDevice(String accessToken, Device device) { return given().spec(requestSpec).body(device) .pathParams("accessToken", accessToken) @@ -479,6 +514,28 @@ public class TestRestClient { .as(User.class); } + public UserId createUserAndLogin(User user, String password) { + UserId userId = postUser(user).getId(); + getUserToken(userId.getId().toString()); + return userId; + } + + public void getUserToken(String id) { + ObjectNode tokenInfo = given().spec(requestSpec) + .get("/api/user/" + id + "/token") + .then() + .extract() + .as(ObjectNode.class); + token = tokenInfo.get("token").asText(); + refreshToken = tokenInfo.get("refreshToken").asText(); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + protected void resetTokens() { + this.token = null; + this.refreshToken = null; + } + public void deleteUser(UserId userId) { given().spec(requestSpec) .delete("/api/user/{userId}", userId.getId()) @@ -643,4 +700,45 @@ public class TestRestClient { } return urlParams; } + + public PageData postEntityDataQuery(EntityDataQuery entityDataQuery) { + return given().spec(requestSpec).body(entityDataQuery) + .post("/api/entitiesQuery/find") + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef<>() {}); + } + + public Long postCountDataQuery(EntityCountQuery entityCountQuery) { + return given().spec(requestSpec).body(entityCountQuery) + .post("/api/entitiesQuery/count") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Long.class); + } + + public Boolean isEdqsApiEnabled() { + return given().spec(requestSpec) + .get("/api/edqs/enabled") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Boolean.class); + } + + public void assignDeviceToCustomer(CustomerId customerId, DeviceId id) { + given().spec(requestSpec) + .post("/api/customer/" + customerId.getId().toString() + "/device/" + id.getId().toString()) + .then() + .statusCode(HTTP_OK); + } + + public void deleteTenant(TenantId tenantId) { + given().spec(requestSpec) + .delete("/api/tenant/" + tenantId.getId().toString()) + .then() + .statusCode(HTTP_OK); + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index aaa4f001c6..75f9779916 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -220,7 +220,8 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true -e TB_EDQS_SYNC_ENABLED=false " + + "tb-core1"); dockerCompose.invokeCompose(); } finally { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java new file mode 100644 index 0000000000..1317011cf0 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -0,0 +1,202 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.edqs; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.ui.utils.EntityPrototypes; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomer; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomerAdmin; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultDeviceProfile; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultTenantAdmin; + +@DisableUIListeners +public class EdqsEntityDataQueryTest extends AbstractContainerTest { + + private TenantId tenantId; + private CustomerId customerId; + private TenantId tenantId2; + private CustomerId customerId2; + private UserId tenantAdminId; + private UserId customerUserId; + private UserId tenant2AdminId; + private UserId customer2UserId; + private final List tenantDevices = new ArrayList<>(); + private final List tenant2Devices = new ArrayList<>(); + private final String deviceProfile = "LoRa-" + RandomStringUtils.randomAlphabetic(10); + + @BeforeClass + public void beforeClass() throws Exception { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> testRestClient.isEdqsApiEnabled()); + + tenantId = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenantAdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId, "tenantAdmin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenantDevices, 97); + customerId = testRestClient.postCustomer(defaultCustomer(tenantId, "Customer")).getId(); + customerUserId = testRestClient.postUser(defaultCustomerAdmin(tenantId, customerId, "customerUser@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId, tenantDevices, 12); + + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + tenantId2 = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenant2AdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId2, "tenant2Admin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenant2Devices, 97); + customerId2 = testRestClient.postCustomer(defaultCustomer(tenantId2, "Customer")).getId(); + customer2UserId = testRestClient.postUser(defaultCustomerAdmin(tenantId2, customerId2, "customer2User@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId2, tenant2Devices, 12); + } + + @BeforeMethod + public void beforeMethod() { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + } + + @AfterClass + public void afterClass() { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.deleteTenant(tenantId); + testRestClient.deleteTenant(tenantId2); + } + + @Test + public void testSysAdminCountEntitiesByQuery() { + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery query = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).compareTo(97L * 2) >= 0); + + testRestClient.getUserToken(tenantAdminId.getId().toString()); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getUserToken(tenant2AdminId.getId().toString()); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + } + + @Test + public void testRetrieveTenantDevicesByDeviceTypeFilter() { + // login tenant admin + testRestClient.getUserToken(tenantAdminId.getId().toString()); + checkUserDevices(tenantDevices); + + // login customer user + testRestClient.getUserToken(customerUserId.getId().toString()); + checkUserDevices(tenantDevices.subList(0, 12)); + + // login other tenant admin + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getUserToken(tenant2AdminId.getId().toString()); + checkUserDevices(tenant2Devices); + } + + private void checkUserDevices(List devices) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of(deviceProfile)); + filter.setDeviceNameFilter(""); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestFields = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestFields, null); + + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery countQuery = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(countQuery).intValue() == devices.size()); + + PageData result = testRestClient.postEntityDataQuery(query); + assertThat(result.getTotalElements()).isEqualTo(devices.size()); + List retrievedDevices = result.getData(); + + assertThat(retrievedDevices).hasSize(10); + List retrievedDeviceNames = retrievedDevices.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).toList(); + assertThat(retrievedDeviceNames).containsExactlyInAnyOrderElementsOf(devices.stream().map(Device::getName).toList().subList(0, 10)); + } + + private String createDevices(String deviceType, List tenantDevices, int deviceCount) throws InterruptedException { + String prefix = StringUtils.randomAlphabetic(5); + for (int i = 0; i < deviceCount; i++) { + Device device = new Device(); + device.setName(prefix + "Device" + i); + device.setType(deviceType); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + //TO make sure devices have different created time + Thread.sleep(1); + String token = RandomStringUtils.randomAlphabetic(10); + Device saved = testRestClient.postDevice(token, device); + tenantDevices.add(saved); + + // save timeseries data + testRestClient.postTelemetry(token, createDeviceTelemetry(i)); + } + return deviceType; + } + + private void assignDevicesToCustomer(CustomerId customerId, List devices, int deviceCount) { + for (int i = 0; i < deviceCount; i++) { + Device device = devices.get(i); + testRestClient.assignDeviceToCustomer(customerId, device.getId()); + } + } + + protected ObjectNode createDeviceTelemetry(int temperature) { + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put("temperature", temperature); + return objectNode; + } + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java index d16cbe35e7..e1c684ddf9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -37,12 +38,26 @@ import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfilePr import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.security.Authority; public class EntityPrototypes { + public static Tenant defaultTenantPrototype(String tenantName) { + Tenant tenant = new Tenant(); + tenant.setTitle(tenantName); + return tenant; + } + + public static Customer defaultCustomer(TenantId tenantId, String title) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(title); + return customer; + } + public static Customer defaultCustomerPrototype(String entityName) { Customer customer = new Customer(); customer.setTitle(entityName); @@ -169,6 +184,23 @@ public class EntityPrototypes { return user; } + public static User defaultTenantAdmin(TenantId tenantId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setEmail(email); + user.setAuthority(Authority.TENANT_ADMIN); + return user; + } + + public static User defaultCustomerAdmin(TenantId tenantId, CustomerId customerId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setCustomerId(customerId); + user.setEmail(email); + user.setAuthority(Authority.CUSTOMER_USER); + return user; + } + public static User defaultUser(String email, CustomerId customerId, String name) { User user = new User(); user.setEmail(email); From a6a6112ecc81092add19940df5d709eb0d35f893 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 24 Feb 2025 12:11:05 +0200 Subject: [PATCH 26/51] Cleanup for EDQS --- .../service/edqs/DefaultEdqsService.java | 8 +-- .../src/main/resources/thingsboard.yml | 10 +++ common/edqs/pom.xml | 6 +- .../server/edqs/repo/TenantRepo.java | 17 ++--- .../edqs/state/KafkaEdqsStateService.java | 1 - .../server/queue/edqs/EdqsQueue.java | 4 +- .../queue/environment/DistributedLock.java | 2 +- .../environment/DistributedLockService.java | 2 +- .../DummyDistributedLockService.java | 6 +- .../environment/ZkDistributedLockService.java | 6 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 2 +- .../common/stats/DummyMessagesStats.java | 54 --------------- .../thingsboard/common/util/JacksonUtil.java | 4 -- .../attributes/CachedAttributesService.java | 2 +- .../server/dao/entity/BaseEntityService.java | 6 +- .../server/dao/entity/EntityDaoRegistry.java | 13 ++-- .../server/dao/model/ModelConstants.java | 2 - .../dao/model/sql/AlarmTypeCompositeKey.java | 33 --------- .../server/dao/sql/rule/JpaRuleNodeDao.java | 5 -- .../dao/sql/rule/RuleNodeRepository.java | 3 - .../dao/sql/user/JpaUserAuthSettingsDao.java | 16 ----- .../widget/WidgetsBundleWidgetRepository.java | 7 -- .../sqlts/latest/TsKvLatestRepository.java | 7 -- edqs/pom.xml | 48 ------------- edqs/src/main/resources/edqs.yml | 67 +++---------------- 25 files changed, 53 insertions(+), 278 deletions(-) delete mode 100644 common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index a6a9b9bb23..92905dc336 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -55,9 +55,9 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.edqs.util.EdqsConverter; import org.thingsboard.server.edqs.processor.EdqsProducer; import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.edqs.util.EdqsConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.EdqsRequestMsg; @@ -103,7 +103,7 @@ public class DefaultEdqsService implements EdqsService { private EdqsProducer eventsProducer; private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; private ExecutorService executor; - private DistributedLock syncLock; + private DistributedLock syncLock; @PostConstruct private void init() { @@ -313,14 +313,14 @@ public class DefaultEdqsService implements EdqsService { .flatMap(KvEntry::getJsonValue) .map(value -> JacksonUtil.fromString(value, EdqsSyncState.class)) .orElse(null); - log.info("getSyncState = {}", state); + log.info("EDQS sync state: {}", state); return state; } @SneakyThrows private void saveSyncState(EdqsSyncStatus status) { EdqsSyncState state = new EdqsSyncState(status); - log.info("saveSyncState {}", state); + log.info("New EDQS sync state: {}", state); attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry( new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)), System.currentTimeMillis())).get(30, TimeUnit.SECONDS); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 46ab8f25de..e12c6aee2d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1586,6 +1586,16 @@ queue: - key: max.poll.records # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" + edqs.events: + # Key-value properties for Kafka consumer for edqs.events topic + - key: max.poll.records + # Max poll records for edqs.events topic + value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}" + edqs.state: + # Key-value properties for Kafka consumer for edqs.state topic + - key: max.poll.records + # Max poll records for edqs.state topic + value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml index 2abd1286b8..e5f5b759ca 100644 --- a/common/edqs/pom.xml +++ b/common/edqs/pom.xml @@ -27,7 +27,7 @@ edqs jar - Thingsboard Server EDQS API + ThingsBoard EDQS API https://thingsboard.io @@ -72,10 +72,6 @@ org.springframework.boot spring-boot-starter-web - - org.apache.kafka - kafka-clients - com.github.ben-manes.caffeine caffeine diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 9003eb4da2..aaf91bc37b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -84,7 +84,7 @@ import static org.thingsboard.server.edqs.util.RepositoryUtils.resolveEntityType @Slf4j public class TenantRepo { - public static final Comparator> CREATED_TIME_COMPARATOR = Comparator.comparingLong(o -> o.getFields() != null ? o.getFields().getCreatedTime() : 0); // FIXME: fields may be null at first + public static final Comparator> CREATED_TIME_COMPARATOR = Comparator.comparingLong(ed -> ed.getFields().getCreatedTime()); public static final Comparator> CREATED_TIME_AND_ID_COMPARATOR = CREATED_TIME_COMPARATOR .thenComparing(EntityData::getId); public static final Comparator> CREATED_TIME_AND_ID_DESC_COMPARATOR = CREATED_TIME_AND_ID_COMPARATOR.reversed(); @@ -186,7 +186,11 @@ public class TenantRepo { EntityData entityData = getOrCreate(entityType, entityId); processFields(fields); - entityData.setFields(entity.getFields()); + EntityFields oldFields = entityData.getFields(); + entityData.setFields(fields); + if (oldFields == null) { + getEntitySet(entityType).add(entityData); + } UUID newCustomerId = fields.getCustomerId(); UUID oldCustomerId = entityData.getCustomerId(); @@ -291,7 +295,6 @@ public class TenantRepo { return getEntityMap(entityType).computeIfAbsent(entityId, id -> { log.debug("[{}] Adding {} {}", tenantId, entityType, id); EntityData entityData = constructEntityData(entityType, entityId); - getEntitySet(entityType).add(entityData); edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); return entityData; }); @@ -331,9 +334,6 @@ public class TenantRepo { EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery); log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query); QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); - if (ctx == null) { - return PageData.emptyPageData(); - } EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); return sortAndConvert(query, queryProcessor.processQuery(), ctx); } @@ -342,9 +342,6 @@ public class TenantRepo { EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery); log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query); QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); - if (ctx == null) { - return 0; - } EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); return queryProcessor.count(); } @@ -383,7 +380,7 @@ public class TenantRepo { // IMPLEMENTATION THAT IS BASED ON TIM SORT (For offset + query.getPageSize() > totalSize / 2) // data.sort(comparator); // var result = data.subList(offset, endIndex); - log.debug("EDQ Sorted in {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTs)); + log.trace("EDQ Sorted in {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTs)); return new PageData<>(toQueryResult(result, query, ctx), totalPages, totalSize, totalSize > requiredSize); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 05eebb188c..c9bae14265 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -169,7 +169,6 @@ public class KafkaEdqsStateService implements EdqsStateService { ready = true; // once true - always true, not to change readiness status on each repartitioning } } - log.error("ready: {}", ready); return ready != null && ready; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java index c773ea4e93..45126186a3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java @@ -24,8 +24,8 @@ public enum EdqsQueue { STATE("edqs.state", true, true); private final String topic; - private final boolean readFromBeginning; // read from the beginning of the topic, instead of the latest committed offset - private final boolean stopWhenRead; // stop consuming when reached an empty msg pack + private final boolean readFromBeginning; + private final boolean stopWhenRead; EdqsQueue(String topic, boolean readFromBeginning, boolean stopWhenRead) { this.topic = topic; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java index b69b98dd4f..9e4b681dab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.queue.environment; -public interface DistributedLock { +public interface DistributedLock { void lock(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java index 61a21cca7c..17b0f84978 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java @@ -17,6 +17,6 @@ package org.thingsboard.server.queue.environment; public interface DistributedLockService { - DistributedLock getLock(String key); + DistributedLock getLock(String key); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java index 03b6d0f2d9..d289c5cb34 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java @@ -27,12 +27,12 @@ import java.util.concurrent.locks.ReentrantLock; public class DummyDistributedLockService implements DistributedLockService { @Override - public DistributedLock getLock(String key) { - return new DummyDistributedLock<>(); + public DistributedLock getLock(String key) { + return new DummyDistributedLock(); } @RequiredArgsConstructor - private static class DummyDistributedLock implements DistributedLock { + private static class DummyDistributedLock implements DistributedLock { private final ReentrantLock lock = new ReentrantLock(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java index 07b8b503f9..c0b9b4d8d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java @@ -33,12 +33,12 @@ public class ZkDistributedLockService implements DistributedLockService { private final ZkDiscoveryService zkDiscoveryService; @Override - public DistributedLock getLock(String key) { - return new ZkDistributedLock<>(key); + public DistributedLock getLock(String key) { + return new ZkDistributedLock(key); } @RequiredArgsConstructor - private class ZkDistributedLock implements DistributedLock { + private class ZkDistributedLock implements DistributedLock { private final InterProcessLock interProcessLock; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 9b580c762e..6d4ed1073c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -53,7 +53,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final String groupId; private final boolean readFromBeginning; // reset offset to beginning - private final boolean stopWhenRead; // stop consuming when reached an empty msg pack + private final boolean stopWhenRead; // stop consuming when reached end offset remembered on start private int readCount; private Map endOffsets; // needed if stopWhenRead is true diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java deleted file mode 100644 index 7847860dcd..0000000000 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.stats; - -public class DummyMessagesStats implements MessagesStats { - @Override - public void incrementTotal(int amount) { - - } - - @Override - public void incrementSuccessful(int amount) { - - } - - @Override - public void incrementFailed(int amount) { - - } - - @Override - public int getTotal() { - return 0; - } - - @Override - public int getSuccessful() { - return 0; - } - - @Override - public int getFailed() { - return 0; - } - - @Override - public void reset() { - - } - -} diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 8dc98377d3..fe0e019537 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -559,10 +559,6 @@ public class JacksonUtil { } } - public static JsonNode getValueByPath(ObjectNode node, String path) { - return node.at("/" + path.replace(".", "/")); - } - @Data public static class JsonNodeProcessingTask { private final String path; diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 3c12ae3209..63371d97e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -281,7 +281,7 @@ public class CachedAttributesService implements AttributesService { String key = deleted.getValue(); if (scope != null && key != null) { cache.evict(new AttributeCacheKey(scope, entityId, key)); - // FIXME: version is Long.MAX_VALUE because we expect that the entity is deleted and there won't be any attributes after this + // using version as Long.MAX_VALUE because we expect that the entity is deleted and there won't be any attributes after this edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); } }); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 1a0f3be1e7..fb8078dd99 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -95,7 +95,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); - if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + if (edqsService.isApiEnabled() && validForEdqs(query)) { EdqsRequest request = EdqsRequest.builder() .entityCountQuery(query) .build(); @@ -112,7 +112,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); - if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + if (edqsService.isApiEnabled() && validForEdqs(query)) { EdqsRequest request = EdqsRequest.builder() .entityDataQuery(query) .build(); @@ -135,7 +135,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return new PageData<>(result, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); } - private boolean validForEdqs(EntityCountQuery query) { + private boolean validForEdqs(EntityCountQuery query) { // for compatibility with PE return true; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 86887e94ec..46e6a51830 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -29,18 +29,19 @@ import java.util.Map; @SuppressWarnings({"unchecked"}) public class EntityDaoRegistry { - private final Map> entityDaos = new EnumMap<>(EntityType.class); + private final Map> daos = new EnumMap<>(EntityType.class); - private EntityDaoRegistry(List> entityDaos) { - entityDaos.forEach(dao -> { - if (dao.getEntityType() != null) { - this.entityDaos.put(dao.getEntityType(), dao); + private EntityDaoRegistry(List> daos) { + daos.forEach(dao -> { + EntityType entityType = dao.getEntityType(); + if (entityType != null) { + this.daos.put(entityType, dao); } }); } public Dao getDao(EntityType entityType) { - Dao dao = (Dao) entityDaos.get(entityType); + Dao dao = (Dao) daos.get(entityType); if (dao == null) { throw new IllegalArgumentException("Missing dao for entity type " + entityType); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index f762be0429..d404c3128c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -281,8 +281,6 @@ public class ModelConstants { public static final String ALARM_COMMENT_TYPE = "type"; public static final String ALARM_COMMENT_COMMENT = "comment"; - public static final String ALARM_TYPES_TABLE_NAME = "alarm_types"; - /** * Entity relation constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java deleted file mode 100644 index 60f5232142..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.model.sql; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.UUID; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class AlarmTypeCompositeKey implements Serializable { - - private UUID tenantId; - private String type; - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index 43087b8652..3bf9ab2963 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -107,11 +107,6 @@ public class JpaRuleNodeDao extends JpaAbstractDao imp ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); } - @Override - public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { - return DaoUtil.toPageData(ruleNodeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); - } - @Override public EntityType getEntityType() { return EntityType.RULE_NODE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index bf19d61a9d..264ea88326 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -63,7 +63,4 @@ public interface RuleNodeRepository extends JpaRepository @Query("DELETE FROM RuleNodeEntity e where e.id in :ids") void deleteByIdIn(@Param("ids") List ids); - @Query("SELECT n FROM RuleNodeEntity n WHERE n.ruleChainId IN (SELECT rc.id FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId)") - Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java index 32b6acf08b..307cadce91 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java @@ -18,12 +18,8 @@ package org.thingsboard.server.dao.sql.user; import lombok.RequiredArgsConstructor; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserAuthSettings; -import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoFaSettings; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; @@ -50,18 +46,6 @@ public class JpaUserAuthSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { - PageData data = DaoUtil.toPageData(repository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); - data.getData().forEach(settings -> { - AccountTwoFaSettings twoFaSettings = settings.getTwoFaSettings(); - if (twoFaSettings != null && twoFaSettings.getConfigs() != null) { - twoFaSettings.getConfigs().values().forEach(config -> config.setSerializeHiddenFields(true)); - } - }); - return data; - } - @Override protected Class getEntityClass() { return UserAuthSettingsEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java index 3a024e1687..89533106f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java @@ -15,11 +15,7 @@ */ package org.thingsboard.server.dao.sql.widget; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetEntity; @@ -30,7 +26,4 @@ public interface WidgetsBundleWidgetRepository extends JpaRepository findAllByWidgetsBundleId(UUID widgetsBundleId); - @Query("SELECT w FROM WidgetsBundleWidgetEntity w WHERE w.widgetsBundleId IN (SELECT b.id FROM WidgetsBundleEntity b WHERE b.tenantId = :tenantId)") - Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 2c97ba30ba..8b44e2c9ad 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -15,12 +15,9 @@ */ package org.thingsboard.server.dao.sqlts.latest; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; @@ -44,10 +41,6 @@ public interface TsKvLatestRepository extends JpaRepository findAllKeysByEntityIds(@Param("entityIds") List entityIds); - @Query("SELECT e FROM TsKvLatestEntity e ORDER BY e.entityId ASC, e.key ASC") - Page findAll(Pageable pageable); - - @Query(value = "SELECT entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version FROM ts_kv_latest WHERE (entity_id, key) > " + "(:entityId, :key) ORDER BY entity_id, key LIMIT :batchSize", nativeQuery = true) List findNextBatch(@Param("entityId") UUID entityId, diff --git a/edqs/pom.xml b/edqs/pom.xml index eb824cb77f..8eb6edfb23 100644 --- a/edqs/pom.xml +++ b/edqs/pom.xml @@ -28,8 +28,6 @@ ThingsBoard Entity Data Query Service Application https://thingsboard.io - ThingsBoard Professional Edition: IoT Platform - Device management, data collection, processing and visualization - UTF-8 @@ -90,16 +88,6 @@ exe provided - - org.thingsboard - tools - test - - - org.springframework.security - spring-security-test - test - org.springframework.boot spring-boot-starter-test @@ -115,42 +103,6 @@ awaitility test - - org.dbunit - dbunit - test - - - com.github.springtestdbunit - spring-test-dbunit - test - - - org.testcontainers - cassandra - test - - - org.testcontainers - postgresql - test - - - org.testcontainers - jdbc - test - - - org.java-websocket - Java-WebSocket - test - - - org.eclipse.milo - sdk-server - ${milo.version} - test - org.assertj assertj-core diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 3d30c5c6fa..80771c2265 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -132,29 +132,17 @@ queue: # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params consumer-properties-per-topic: - tb_ota_package: - # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + edqs.events: + # Key-value properties for Kafka consumer for edqs.events topic - key: max.poll.records - # Example of specific consumer properties value per topic - value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" - tb_version_control: - # Example of specific consumer properties value per topic for VC - - key: max.poll.interval.ms - # Example of specific consumer properties value per topic for VC - value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" - # tb_rule_engine.sq: - # - key: max.poll.records - # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" - tb_housekeeper: - # Consumer properties for Housekeeper tasks topic + # Max poll records for edqs.events topic + value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}" + edqs.state: + # Key-value properties for Kafka consumer for edqs.state topic - key: max.poll.records - # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one - value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_MAX_POLL_RECORDS:1}" - tb_housekeeper.reprocessing: - # Consumer properties for Housekeeper reprocessing topic - - key: max.poll.records - # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one - value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" + # Max poll records for edqs.state topic + value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -162,26 +150,6 @@ queue: # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: - # Kafka properties for Rule Engine - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - # Kafka properties for Core topics - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - # Kafka properties for Transport Api topics - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - # Kafka properties for Notifications topics - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - # Kafka properties for JS Executor topics - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" - # Kafka properties for OTA updates topic - ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - # Kafka properties for Version Control topic - version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - # Kafka properties for Integration Api topics - integration-api: "${TB_QUEUE_KAFKA_INTEGRATION_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - # Kafka properties for Housekeeper tasks topic - housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time - housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions @@ -197,23 +165,6 @@ queue: kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 - transport_api: - # Topic used to consume api requests from transport microservices - requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" - # Topic used to produce api responses to transport microservices - responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" - # Maximum pending api requests from transport microservices to be handled by server - max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" - # Maximum timeout in milliseconds to handle api request from transport microservice by server - max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" - # Amount of threads used to invoke callbacks - max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" - # Amount of threads used for transport API requests - max_core_handler_threads: "${TB_QUEUE_TRANSPORT_MAX_CORE_HANDLER_THREADS:16}" - # Interval in milliseconds to poll api requests from transport microservices - request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" - # Interval in milliseconds to poll api response from transport microservices - response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" # General service parameters service: From f28f14f2fc1f0ebc35c6b686c1ab91cf460e546f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 24 Feb 2025 17:36:42 +0200 Subject: [PATCH 27/51] minor refactoring --- .../controller/EntityQueryControllerTest.java | 3 ++- docker/edqs.env | 1 - .../server/msa/ContainerTestSuite.java | 6 ++--- .../server/msa/TestRestClient.java | 22 ------------------- 4 files changed, 4 insertions(+), 28 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index d9652946ab..757f58b457 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -713,7 +713,8 @@ public class EntityQueryControllerTest extends AbstractControllerTest { // all devices with ownerName = TEST TENANT EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); - countByQueryAndCheck(query, numOfDevices); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == numOfDevices); // all devices with ownerName = TEST TENANT EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); diff --git a/docker/edqs.env b/docker/edqs.env index 3c91c92e54..33e64d0c96 100644 --- a/docker/edqs.env +++ b/docker/edqs.env @@ -2,6 +2,5 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 TB_KAFKA_SERVERS=kafka:9092 -TB_EDQS_STATS_ENABLED=false METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index d22861e286..5491331d65 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -164,10 +164,8 @@ public class ContainerTestSuite { } // to trigger edqs synchronization - if (true) { - addToFile(targetDir, "tb-node.env", - Map.of("TB_EDQS_SYNC_ENABLED", "true")); - } + addToFile(targetDir, "tb-node.env", + Map.of("TB_EDQS_SYNC_ENABLED", "true")); testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 3c61c65ef1..4e6825f2a9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -114,28 +114,6 @@ public class TestRestClient { requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); } - public JsonNode getActivateRequest(String password) { - ObjectNode response = given().spec(requestSpec).get("/api/noauth/activate?activateToken={activateToken}", token) - .then() - .statusCode(HTTP_OK) - .extract() - .as(ObjectNode.class); - return JacksonUtil.newObjectNode() - .put("activateToken", this.token) - .put("password", password); - } - - public void activateAndLoginAsUser(JsonNode activateRequest) { - ObjectNode tokenInfo = given().spec(requestSpec).body(activateRequest) - .post("/api/noauth/activate") - .then() - .extract() - .as(ObjectNode.class); - token = tokenInfo.get("token").asText(); - refreshToken = tokenInfo.get("refreshToken").asText(); - requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); - } - public Tenant postTenant(Tenant tenant) { return given().spec(requestSpec).body(tenant) .post("/api/tenant") From 8eabe34ac6eb7297884bfa6b368640debc99ab8f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 25 Feb 2025 11:08:05 +0200 Subject: [PATCH 28/51] edqs docker image renamed --- docker/.env | 2 +- docker/docker-compose.edqs.yml | 4 ++-- .../java/org/thingsboard/server/msa/ContainerTestSuite.java | 4 ++-- .../thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java | 1 - msa/edqs/pom.xml | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docker/.env b/docker/.env index 38724263fc..f6d9ebc0f8 100644 --- a/docker/.env +++ b/docker/.env @@ -14,7 +14,7 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor -EDQS_DOCKER_NAME=edqs +EDQS_DOCKER_NAME=tb-edqs TB_VERSION=latest diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml index f989c12a2e..3a01a3c970 100644 --- a/docker/docker-compose.edqs.yml +++ b/docker/docker-compose.edqs.yml @@ -17,14 +17,14 @@ version: '3.0' services: - edqs-1: + tb-edqs-1: restart: always image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" env_file: - edqs.env depends_on: - zookeeper - edqs-2: + tb-edqs-2: restart: always image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" env_file: diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 5491331d65..45e227ed03 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -186,8 +186,8 @@ public class ContainerTestSuite { .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("edqs-1", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("edqs-2", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-edqs-1", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-2", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java index 1317011cf0..55a5f11821 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -53,7 +53,6 @@ import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustom import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultDeviceProfile; import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultTenantAdmin; -@DisableUIListeners public class EdqsEntityDataQueryTest extends AbstractContainerTest { private TenantId tenantId; diff --git a/msa/edqs/pom.xml b/msa/edqs/pom.xml index 4874c06e4b..7e7a244cc2 100644 --- a/msa/edqs/pom.xml +++ b/msa/edqs/pom.xml @@ -35,7 +35,7 @@ UTF-8 ${basedir}/../.. edqs - edqs + tb-edqs /var/log/${pkg.name} /usr/share/${pkg.name} pre-integration-test From 6c8887995a9b5de7beb3ae1243c6dc9e45bda120 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 25 Feb 2025 13:23:26 +0200 Subject: [PATCH 29/51] fixed license, included edqs tests to test suite --- docker/docker-compose.edqs.yml | 2 +- edqs/src/main/conf/edqs.conf | 2 +- edqs/src/main/conf/logback.xml | 2 +- .../server/msa/ContainerTestSuite.java | 3 ++- .../thingsboard/server/msa/TestRestClient.java | 10 +++++++--- .../msa/edqs/EdqsEntityDataQueryTest.java | 18 +++++++++++------- msa/black-box-tests/src/test/resources/all.xml | 10 ++++++++++ msa/edqs/docker/Dockerfile | 2 +- msa/edqs/docker/start-tb-edqs.sh | 2 +- msa/edqs/pom.xml | 2 +- 10 files changed, 36 insertions(+), 17 deletions(-) diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml index 3a01a3c970..c50740faeb 100644 --- a/docker/docker-compose.edqs.yml +++ b/docker/docker-compose.edqs.yml @@ -1,5 +1,5 @@ # -# Copyright © 2016-2024 The Thingsboard Authors +# Copyright © 2016-2025 The Thingsboard Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/edqs/src/main/conf/edqs.conf b/edqs/src/main/conf/edqs.conf index ae880ef7e8..3f96fd590d 100644 --- a/edqs/src/main/conf/edqs.conf +++ b/edqs/src/main/conf/edqs.conf @@ -1,5 +1,5 @@ # -# Copyright © 2016-2024 The Thingsboard Authors +# Copyright © 2016-2025 The Thingsboard Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/edqs/src/main/conf/logback.xml b/edqs/src/main/conf/logback.xml index 4ff57d48ab..850a28b212 100644 --- a/edqs/src/main/conf/logback.xml +++ b/edqs/src/main/conf/logback.xml @@ -1,7 +1,7 @@ + + + + + /var/log/edqs/${TB_SERVICE_ID}/tb-edqs.log + + /var/log/edqs/tb-edqs.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/docker/tb-node-edqs.env b/docker/tb-node-edqs.env new file mode 100644 index 0000000000..1ef523af50 --- /dev/null +++ b/docker/tb-node-edqs.env @@ -0,0 +1,4 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_SYNC_ENABLED=true +TB_EDQS_API_ENABLED=true \ No newline at end of file diff --git a/docker/tb-node.env b/docker/tb-node.env index d40ca66621..e07d5a2f23 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,10 +4,9 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 JS_EVALUATOR=remote TRANSPORT_TYPE=remote +TB_EDQS_MODE=remote HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus - -TB_EDQS_MODE=remote \ No newline at end of file diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index bc6c3611f5..5b57768b79 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -51,6 +51,7 @@ public class ContainerTestSuite { private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; + private static final String TB_EDQS_LOG_REGEXP = ".*All partitions processed.*"; private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); @@ -115,6 +116,7 @@ public class ContainerTestSuite { List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.edqs.yml"), + new File(targetDir + "docker-compose.edqs.volumes.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.mosquitto.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), @@ -163,11 +165,6 @@ public class ContainerTestSuite { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); } - // to trigger edqs synchronization - addToFile(targetDir, "tb-node.env", - Map.of("TB_EDQS_SYNC_ENABLED", "true", - "TB_EDQS_API_ENABLED", "true")); - testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) .withLocalCompose(true) @@ -189,8 +186,8 @@ public class ContainerTestSuite { .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-edqs-1", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-edqs-2", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-edqs-1", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-2", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 48664716f5..9ce9f82bca 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -48,6 +48,7 @@ public class ThingsBoardDbInstaller { private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; + private final static String TB_EDQS_LOG_VOLUME = "tb-edqs-log-test-volume"; private final static String JAVA_OPTS = "-Xmx512m"; private final DockerComposeExecutor dockerCompose; @@ -65,6 +66,7 @@ public class ThingsBoardDbInstaller { private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; private final String tbVcExecutorLogVolume; + private final String tbEdqsLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -103,6 +105,7 @@ public class ThingsBoardDbInstaller { tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; + tbEdqsLogVolume = project + "_" + TB_EDQS_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -119,6 +122,7 @@ public class ThingsBoardDbInstaller { env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + env.put("TB_EDQS_LOG_VOLUME", tbEdqsLogVolume); if (IS_REDIS_CLUSTER) { for (int i = 0; i < 6; i++) { env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); @@ -189,6 +193,9 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbEdqsLogVolume); + dockerCompose.invokeDocker(); + StringBuilder additionalServices = new StringBuilder(); if (IS_HYBRID_MODE) { additionalServices.append(" cassandra"); @@ -241,6 +248,7 @@ public class ThingsBoardDbInstaller { copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); + copyLogs(tbEdqsLogVolume, "./target/tb-edqs-logs/"); StringJoiner rmVolumesCommand = new StringJoiner(" ") .add("volume rm -f") @@ -252,6 +260,7 @@ public class ThingsBoardDbInstaller { .add(tbMqttTransportLogVolume) .add(tbSnmpTransportLogVolume) .add(tbVcExecutorLogVolume) + .add(tbEdqsLogVolume) .add(resolveRedisComposeVolumeLog()); if (IS_HYBRID_MODE) { From aa0179303cbe94252777d98b8b7a1579ad49aeda Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 27 Feb 2025 12:45:22 +0200 Subject: [PATCH 36/51] Subscribe to Zookeeper events only after application starts Sometimes ZK events came before the app start, causing repartition change events, while event listeners are not yet initialized by Spring --- .../server/queue/discovery/ZkDiscoveryService.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index ad2a97de52..3ad6164c61 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -143,6 +143,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } else { log.info("Received application ready event. Starting current ZK node."); } + subscribeToEvents(); if (client.getState() != CuratorFrameworkState.STARTED) { log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); return; @@ -212,6 +213,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi try { destroyZkClient(); initZkClient(); + subscribeToEvents(); publishCurrentServer(); } catch (Exception e) { log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); @@ -227,7 +229,6 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi client.start(); client.blockUntilConnected(); cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); cache.start(); stopped = false; log.info("ZK client connected"); @@ -239,6 +240,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } + private void subscribeToEvents() { + cache.getListenable().addListener(this); + } + private void unpublishCurrentServer() { try { if (nodePath != null) { From 16169dc7285ea2758bbecbfe0377772367f2923f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 27 Feb 2025 15:58:26 +0200 Subject: [PATCH 37/51] Improve ZkDiscoveryService shutdown --- .../server/queue/discovery/ZkDiscoveryService.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 3ad6164c61..f7a4d2abf6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -251,25 +251,21 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } catch (Exception e) { log.error("Failed to delete ZK node {}", nodePath, e); - throw new RuntimeException(e); } } private void destroyZkClient() { stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) { - } + unpublishCurrentServer(); CloseableUtils.closeQuietly(cache); CloseableUtils.closeQuietly(client); log.info("ZK client disconnected"); } @PreDestroy - public void destroy() { - destroyZkClient(); + private void destroy() { zkExecutorService.shutdownNow(); + destroyZkClient(); log.info("Stopped discovery service"); } From 0115ac231b19613e8133a26c9c6cf1a614fb6458 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 27 Feb 2025 16:04:56 +0200 Subject: [PATCH 38/51] Fix license headers --- .../common/data/queue/ProcessingStrategyType.java | 15 --------------- .../server/edqs/repo/AssetTypeFilterTest.java | 15 --------------- 2 files changed, 30 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java index 5b8c86d7d1..ca63d34a84 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java @@ -13,21 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.common.data.queue; public enum ProcessingStrategyType { diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java index 1420dfac0f..0961a7c12c 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java @@ -13,21 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.edqs.repo; import org.junit.After; From 02857b70bcdd2149392ca6ed1717c2847be896f9 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 27 Feb 2025 17:31:46 +0200 Subject: [PATCH 39/51] fixed flaky test, code refactoring --- .../subscription/TbAlarmDataSubCtx.java | 3 +- .../entitiy/EdqsEntityServiceTest.java | 7 +++ .../service/entitiy/EntityServiceTest.java | 56 ++++++++----------- .../server/dao/entity/BaseEntityService.java | 2 +- .../server/msa/TestRestClient.java | 15 ++++- .../msa/edqs/EdqsEntityDataQueryTest.java | 21 +++++-- 6 files changed, 59 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java index c471c84b11..f6b4067543 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java @@ -39,6 +39,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; @@ -359,7 +360,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx { EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityDataSortOrder entitiesSortOrder; if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { - entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME)); } else { entitiesSortOrder = sortOrder; } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index b9cd7dbac2..f4cdf3038b 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; @@ -106,6 +107,12 @@ public class EdqsEntityServiceTest extends EntityServiceTest { result -> result.getTotalElements() == expectedResultSize); } + @Override + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), + loadedTelemetry -> loadedTelemetry.containsAll(expectedTelemetries)); + } + @Override protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 0297281b68..76d3ac1659 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -116,7 +116,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @@ -1571,47 +1570,19 @@ public class EntityServiceTest extends AbstractControllerTest { for (EntityKeyType currentAttributeKeyType : attributesEntityTypes) { List latestValues = Collections.singletonList(new EntityKey(currentAttributeKeyType, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = findByQueryAndCheck(query, 67); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(67, loadedEntities.size()); - List loadedTemperatures = new ArrayList<>(); - for (Device device : devices) { - loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) - .getLatest().get(currentAttributeKeyType).get("temperature").getValue()); - } - List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - assertThat(loadedTemperatures).containsExactlyInAnyOrderElementsOf(deviceTemperatures); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).toList(); + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45); List keyFiltersHighTemperature = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersHighTemperature); - data = findByQueryAndCheck(query, highTemperatures.size()); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList()); - List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - assertThat(loadedHighTemperatures).containsExactlyInAnyOrderElementsOf(deviceHighTemperatures); + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", highTemperatures.stream().map(Object::toString).toList()); } deviceService.deleteDevicesByTenantId(tenantId); } - @Test public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { @@ -2519,7 +2490,7 @@ public class EntityServiceTest extends AbstractControllerTest { findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); } - private PageData findByQuery(EntityDataQuery query) { + protected PageData findByQuery(EntityDataQuery query) { return findByQuery(new CustomerId(CustomerId.NULL_UUID), query); } @@ -2527,7 +2498,7 @@ public class EntityServiceTest extends AbstractControllerTest { return entityService.findEntityDataByQuery(tenantId, customerId, query); } - private PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { + protected PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { return findByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), query, expectedResultSize); } @@ -2537,6 +2508,23 @@ public class EntityServiceTest extends AbstractControllerTest { return result; } + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetry) { + List entitiesTelemetry = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry); + assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry); + return entitiesTelemetry; + } + + protected List findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + PageData data = findByQueryAndCheck(query, expectedTelemetries.size()); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } + return loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList(); + } + protected long countByQuery(CustomerId customerId, EntityCountQuery query) { return entityService.countEntitiesByQuery(tenantId, customerId, query); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index bbe21341a4..3e398f9746 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -112,7 +112,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); - if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { + if (edqsApiService.isEnabled() && validForEdqs(query)) { EdqsRequest request = EdqsRequest.builder() .entityDataQuery(query) .build(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index ed4b54f2fc..d74438a428 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -237,6 +237,15 @@ public class TestRestClient { .as(JsonNode.class); } + public JsonNode getLatestTelemetry(EntityId entityId) { + return given().spec(requestSpec) + .get("/api/plugins/telemetry/" + entityId.getEntityType().name() + "/" + entityId.getId() + "/values/timeseries") + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + public JsonPath postProvisionRequest(String provisionRequest) { return given().spec(requestSpec) .body(provisionRequest) @@ -498,13 +507,13 @@ public class TestRestClient { public UserId createUserAndLogin(User user, String password) { UserId userId = postUser(user).getId(); - getAndSetUserToken(userId.getId().toString()); + getAndSetUserToken(userId); return userId; } - public void getAndSetUserToken(String id) { + public void getAndSetUserToken(UserId id) { ObjectNode tokenInfo = given().spec(requestSpec) - .get("/api/user/" + id + "/token") + .get("/api/user/" + id.getId().toString() + "/token") .then() .extract() .as(ObjectNode.class); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java index 6c0a507431..8e77b315e6 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.msa.AbstractContainerTest; import org.thingsboard.server.msa.DisableUIListeners; import org.thingsboard.server.msa.ui.utils.EntityPrototypes; @@ -44,6 +45,7 @@ import org.thingsboard.server.msa.ui.utils.EntityPrototypes; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -111,16 +113,16 @@ public class EdqsEntityDataQueryTest extends AbstractContainerTest { EntityCountQuery query = new EntityCountQuery(allDeviceFilter); await("Waiting for total device count") .atMost(30, TimeUnit.SECONDS) - .until(() -> testRestClient.postCountDataQuery(query).compareTo(97L * 2) >= 0); + .until(() -> testRestClient.postCountDataQuery(query).equals(97L * 2)); - testRestClient.getAndSetUserToken(tenantAdminId.getId().toString()); + testRestClient.getAndSetUserToken(tenantAdminId); await("Waiting for total device count") .atMost(30, TimeUnit.SECONDS) .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); testRestClient.resetToken(); testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); - testRestClient.getAndSetUserToken(tenant2AdminId.getId().toString()); + testRestClient.getAndSetUserToken(tenant2AdminId); await("Waiting for total device count") .atMost(30, TimeUnit.SECONDS) .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); @@ -129,17 +131,17 @@ public class EdqsEntityDataQueryTest extends AbstractContainerTest { @Test public void testRetrieveTenantDevicesByDeviceTypeFilter() { // login tenant admin - testRestClient.getAndSetUserToken(tenantAdminId.getId().toString()); + testRestClient.getAndSetUserToken(tenantAdminId); checkUserDevices(tenantDevices); // login customer user - testRestClient.getAndSetUserToken(customerUserId.getId().toString()); + testRestClient.getAndSetUserToken(customerUserId); checkUserDevices(tenantDevices.subList(0, 12)); // login other tenant admin testRestClient.resetToken(); testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); - testRestClient.getAndSetUserToken(tenant2AdminId.getId().toString()); + testRestClient.getAndSetUserToken(tenant2AdminId); checkUserDevices(tenant2Devices); } @@ -168,6 +170,13 @@ public class EdqsEntityDataQueryTest extends AbstractContainerTest { assertThat(retrievedDevices).hasSize(10); List retrievedDeviceNames = retrievedDevices.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).toList(); assertThat(retrievedDeviceNames).containsExactlyInAnyOrderElementsOf(devices.stream().map(Device::getName).toList().subList(0, 10)); + + //check temperature + for (int i = 0; i < 10; i++) { + Map> latest = retrievedDevices.get(i).getLatest(); + String name = latest.get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + //assertThat(latest.get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).isEqualTo(name.substring(name.length() - 1)); + } } private String createDevices(String deviceType, List tenantDevices, int deviceCount) throws InterruptedException { From 45d289d0a3a7c4e5e677b93d2aa2c2bd0b5dd761 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 27 Feb 2025 18:22:52 +0200 Subject: [PATCH 40/51] deleted TENANT_PROFILE from edqs types --- .../org/thingsboard/server/common/data/ObjectType.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index e25ef57a44..51c631fe4f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -60,15 +60,14 @@ public enum ObjectType { LATEST_TS_KV; public static final Set edqsTenantTypes = EnumSet.of( - TENANT, TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + TENANT, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS ); public static final Set edqsTypes = EnumSet.copyOf(edqsTenantTypes); - public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, + public static final Set edqsSystemTypes = EnumSet.of(TENANT, USER, DASHBOARD, API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); public static final Set unversionedTypes = EnumSet.of( - QUEUE_STATS, // created once, never updated - TENANT_PROFILE // only for total count calculation + QUEUE_STATS // created once, never updated ); static { From 506ec363eb935d323f99959e712f7ea2535bfb86 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 28 Feb 2025 12:45:01 +0200 Subject: [PATCH 41/51] EDQS: optimize serialization, fix api usage fields --- .../data/edqs/fields/ApiUsageStateFields.java | 5 +++-- .../common/data/edqs/fields/EntityFields.java | 2 ++ .../common/data/edqs/fields/FieldsUtil.java | 1 + .../server/edqs/repo/TenantRepo.java | 4 +++- .../server/edqs/util/EdqsConverter.java | 17 ++++++++++++++--- .../usagerecord/ApiUsageStateRepository.java | 3 ++- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java index 3524fdbcbf..d10f375bc1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java @@ -43,8 +43,9 @@ public class ApiUsageStateFields extends AbstractEntityFields { public ApiUsageStateFields(UUID id, long createdTime, UUID tenantId, UUID entityId, String entityType, ApiUsageStateValue transportState, ApiUsageStateValue dbStorageState, ApiUsageStateValue reExecState, ApiUsageStateValue jsExecState, ApiUsageStateValue tbelExecState, - ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState) { - super(id, createdTime, tenantId); + ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState, + Long version) { + super(id, createdTime, tenantId, null, null, version); this.entityId = (entityType != null && entityId != null) ? EntityIdFactory.getByTypeAndUuid(entityType, entityId) : null; this.transportState = transportState; this.dbStorageState = dbStorageState; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java index 1bbf8a519a..532c4a92ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.edqs.fields; +import com.fasterxml.jackson.annotation.JsonInclude; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.thingsboard.server.common.data.id.EntityId; @@ -23,6 +24,7 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +@JsonInclude(JsonInclude.Include.NON_NULL) public interface EntityFields { Logger log = LoggerFactory.getLogger(EntityFields.class); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index 72c45066f1..ce9873b5d6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -284,6 +284,7 @@ public class FieldsUtil { .emailExecState(entity.getEmailExecState()) .smsExecState(entity.getSmsExecState()) .alarmExecState(entity.getAlarmExecState()) + .version(entity.getVersion()) .build(); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 5020ed3476..0c55bc50dd 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -219,7 +219,9 @@ public class TenantRepo { EntityType entityType = entity.getType(); EntityData removed = getEntityMap(entityType).remove(entityId); if (removed != null) { - getEntitySet(entityType).remove(removed); + if (removed.getFields() != null) { + getEntitySet(entityType).remove(removed); + } edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); UUID customerId = removed.getCustomerId(); if (customerId != null) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index 781ae2ed3d..5b4cd7ac4a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -15,12 +15,15 @@ */ package org.thingsboard.server.edqs.util; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.google.protobuf.ByteString; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; @@ -217,16 +220,24 @@ public class EdqsConverter { @RequiredArgsConstructor private static class JsonConverter implements Converter { + private static final ObjectMapper mapper = JsonMapper.builder() + .visibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) + .visibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE) + .visibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE) + .build(); + private final Class type; + @SneakyThrows @Override public byte[] serialize(ObjectType objectType, T value) { - return JacksonUtil.writeValueAsBytes(value); + return mapper.writeValueAsBytes(value); } + @SneakyThrows @Override public T deserialize(ObjectType objectType, byte[] bytes) { - return JacksonUtil.fromBytes(bytes, this.type); + return mapper.readValue(bytes, this.type); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index b9afe8ea4c..98e62fc110 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -54,6 +54,7 @@ public interface ApiUsageStateRepository extends JpaRepository :id ORDER BY a.id") + "a.emailExecState, a.smsExecState, a.alarmExecState, a.version) FROM ApiUsageStateEntity a WHERE a.id > :id ORDER BY a.id") List findNextBatch(@Param("id") UUID id, Limit limit); + } From d25af150749816a53459433451c14d148ec2e8ca Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 28 Feb 2025 15:15:00 +0200 Subject: [PATCH 42/51] fixed test --- .../thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java index 8e77b315e6..1e3f3b1a60 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -113,7 +113,7 @@ public class EdqsEntityDataQueryTest extends AbstractContainerTest { EntityCountQuery query = new EntityCountQuery(allDeviceFilter); await("Waiting for total device count") .atMost(30, TimeUnit.SECONDS) - .until(() -> testRestClient.postCountDataQuery(query).equals(97L * 2)); + .until(() -> testRestClient.postCountDataQuery(query).compareTo(97L * 2) >= 0); testRestClient.getAndSetUserToken(tenantAdminId); await("Waiting for total device count") From faff0ea8c2ca937deeac1b2b33c73c472abe1ef5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 28 Feb 2025 17:22:53 +0200 Subject: [PATCH 43/51] Refactor TbRocksDb usage --- .../server/service/cf/CfRocksDb.java | 11 ++- .../thingsboard/server/utils/TbRocksDb.java | 71 ------------------- .../server/edqs/util/EdqsRocksDb.java | 5 +- .../server/edqs/util/TbRocksDb.java | 31 +++++--- 4 files changed, 33 insertions(+), 85 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java index cb6d3f0d6b..f95227bc24 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -15,22 +15,29 @@ */ package org.thingsboard.server.service.cf; +import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import org.rocksdb.Options; import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.utils.TbRocksDb; +import org.thingsboard.server.edqs.util.TbRocksDb; @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory'") public class CfRocksDb extends TbRocksDb { - public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { + public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) { super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); } + @PostConstruct + @Override + public void init() { + super.init(); + } + @PreDestroy @Override public void close() { diff --git a/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java b/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java deleted file mode 100644 index 5c4fe185e5..0000000000 --- a/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.utils; - -import lombok.SneakyThrows; -import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksIterator; -import org.rocksdb.WriteOptions; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.function.BiConsumer; - -public class TbRocksDb { - - protected final String path; - private final WriteOptions writeOptions; - protected final RocksDB db; - - static { - RocksDB.loadLibrary(); - } - - public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) throws Exception { - this.path = path; - this.writeOptions = writeOptions; - Files.createDirectories(Path.of(path).getParent()); - this.db = RocksDB.open(dbOptions, path); - } - - @SneakyThrows - public void put(String key, byte[] value) { - db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); - } - - public void forEach(BiConsumer processor) { - try (RocksIterator iterator = db.newIterator()) { - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - String key = new String(iterator.key(), StandardCharsets.UTF_8); - processor.accept(key, iterator.value()); - } - } - } - - @SneakyThrows - public void delete(String key) { - db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); - } - - public void close() { - if (db != null) { - db.close(); - } - } - -} \ No newline at end of file diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java index ae1436e6ca..4a991432c7 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java @@ -19,6 +19,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; import org.rocksdb.Options; +import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; @@ -34,16 +35,18 @@ public class EdqsRocksDb extends TbRocksDb { private boolean isNew; public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${user.home}/.rocksdb/edqs}") String path) { - super(path, new Options().setCreateIfMissing(true)); + super(path, new Options().setCreateIfMissing(true), new WriteOptions()); } @PostConstruct + @Override public void init() { isNew = !Files.exists(Path.of(path)); super.init(); } @PreDestroy + @Override public void close() { super.close(); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java index cd5ba0c9d9..23f2fa2c9e 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java @@ -15,35 +15,43 @@ */ package org.thingsboard.server.edqs.util; -import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.rocksdb.Options; import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; +import org.rocksdb.WriteOptions; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.function.BiConsumer; -@RequiredArgsConstructor public class TbRocksDb { protected final String path; - private final Options options; - - private RocksDB db; + private final Options dbOptions; + private final WriteOptions writeOptions; + protected RocksDB db; static { RocksDB.loadLibrary(); } + public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) { + this.path = path; + this.dbOptions = dbOptions; + this.writeOptions = writeOptions; + } + @SneakyThrows public void init() { - db = RocksDB.open(options, path); + Files.createDirectories(Path.of(path).getParent()); + db = RocksDB.open(dbOptions, path); } - public void put(String key, byte[] value) throws RocksDBException { - db.put(key.getBytes(StandardCharsets.UTF_8), value); + @SneakyThrows + public void put(String key, byte[] value) { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); } public void forEach(BiConsumer processor) { @@ -55,8 +63,9 @@ public class TbRocksDb { } } - public void delete(String key) throws RocksDBException { - db.delete(key.getBytes(StandardCharsets.UTF_8)); + @SneakyThrows + public void delete(String key) { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); } public void close() { From 6f21c950197a81349596bf64c9e3419ded4c7ad0 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 28 Feb 2025 17:44:42 +0200 Subject: [PATCH 44/51] fixed docker-compose.edqs.yml --- docker/docker-compose.edqs.yml | 10 ++++++++-- docker/tb-core-edqs.env | 5 +++++ docker/tb-node.env | 1 - docker/{tb-node-edqs.env => tb-rule-engine-edqs.env} | 1 - 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 docker/tb-core-edqs.env rename docker/{tb-node-edqs.env => tb-rule-engine-edqs.env} (80%) diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml index 2f0e836240..67e9c987e3 100644 --- a/docker/docker-compose.edqs.yml +++ b/docker/docker-compose.edqs.yml @@ -19,10 +19,16 @@ version: '3.0' services: tb-core1: env_file: - - tb-node-edqs.env + - tb-core-edqs.env tb-core2: env_file: - - tb-node-edqs.env + - tb-core-edqs.env + tb-rule-engine1: + env_file: + - tb-rule-engine-edqs.env + tb-rule-engine2: + env_file: + - tb-rule-engine-edqs.env tb-edqs-1: restart: always image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" diff --git a/docker/tb-core-edqs.env b/docker/tb-core-edqs.env new file mode 100644 index 0000000000..1417b780ec --- /dev/null +++ b/docker/tb-core-edqs.env @@ -0,0 +1,5 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_MODE=remote +TB_EDQS_SYNC_ENABLED=true +TB_EDQS_API_ENABLED=true \ No newline at end of file diff --git a/docker/tb-node.env b/docker/tb-node.env index e07d5a2f23..85a60eb51a 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,7 +4,6 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 JS_EVALUATOR=remote TRANSPORT_TYPE=remote -TB_EDQS_MODE=remote HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false diff --git a/docker/tb-node-edqs.env b/docker/tb-rule-engine-edqs.env similarity index 80% rename from docker/tb-node-edqs.env rename to docker/tb-rule-engine-edqs.env index 1ef523af50..82395ddcfe 100644 --- a/docker/tb-node-edqs.env +++ b/docker/tb-rule-engine-edqs.env @@ -1,4 +1,3 @@ # ThingsBoard server configuration with enabled EDQS synchronization TB_EDQS_SYNC_ENABLED=true -TB_EDQS_API_ENABLED=true \ No newline at end of file From 8ad3afa4ba1f7b255cd8e550863e75df51011c2e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 28 Feb 2025 17:46:11 +0200 Subject: [PATCH 45/51] unignored test check --- .../thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java index 1e3f3b1a60..53d8a72e7f 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -175,7 +175,7 @@ public class EdqsEntityDataQueryTest extends AbstractContainerTest { for (int i = 0; i < 10; i++) { Map> latest = retrievedDevices.get(i).getLatest(); String name = latest.get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - //assertThat(latest.get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).isEqualTo(name.substring(name.length() - 1)); + assertThat(latest.get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).isEqualTo(name.substring(name.length() - 1)); } } From 9f274aa197923fc844a0b0e742cbf7da509fc08c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 3 Mar 2025 14:54:00 +0200 Subject: [PATCH 46/51] set EDQS_ENABLED to false by default --- docker/.env | 1 + docker/compose-utils.sh | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/.env b/docker/.env index f6d9ebc0f8..71722247df 100644 --- a/docker/.env +++ b/docker/.env @@ -15,6 +15,7 @@ LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor EDQS_DOCKER_NAME=tb-edqs +EDQS_ENABLED=false TB_VERSION=latest diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index bd04657339..5767026b11 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -151,7 +151,6 @@ function permissionList() { 799 799 tb-transports/coap/log 799 799 tb-vc-executor/log 999 999 tb-node/postgres - 799 799 edqs/log " source .env @@ -162,6 +161,12 @@ function permissionList() { " fi + if [ "$EDQS_ENABLED" = true ]; then + PERMISSION_LIST="$PERMISSION_LIST + 799 799 edqs/log + " + fi + CACHE="${CACHE:-redis}" case $CACHE in redis) From bab367c50f26fa5c4b7ee971ffdd7cbad7ab930f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 3 Mar 2025 15:24:23 +0200 Subject: [PATCH 47/51] delete redundant env variable --- .../java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 9ce9f82bca..9c1a90a4f1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -227,7 +227,7 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true -e TB_EDQS_SYNC_ENABLED=false " + + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true " + "tb-core1"); dockerCompose.invokeCompose(); From 964beaae3f1db61d1d98ed0b8492e08a3a9f9936 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 4 Mar 2025 12:07:19 +0200 Subject: [PATCH 48/51] fixed flaky tests --- .../entitiy/EdqsEntityServiceTest.java | 4 +- .../service/entitiy/EntityServiceTest.java | 204 +++--------------- 2 files changed, 36 insertions(+), 172 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index f4cdf3038b..7f29821afd 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -108,9 +108,9 @@ public class EdqsEntityServiceTest extends EntityServiceTest { } @Override - protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { return await().atMost(15, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), - loadedTelemetry -> loadedTelemetry.containsAll(expectedTelemetries)); + loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries)); } @Override diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index dc1f46eb3c..18687cedba 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -413,22 +413,10 @@ public class EntityServiceTest extends AbstractControllerTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = findByQueryAndCheck(query, 25); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(25, loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); - //count query - countByQueryAndCheck(query, 25); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -439,25 +427,10 @@ public class EntityServiceTest extends AbstractControllerTest { highTemperatureFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(highTemperatureFilter); - query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = findByQueryAndCheck(query, highTemperatures.size()); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); - - //count query - countByQueryAndCheck(query, deviceHighTemperatures.size()); + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -590,20 +563,12 @@ public class EntityServiceTest extends AbstractControllerTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = findByQueryAndCheck(query, 25); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(25, loadedEntities.size()); + List loadedEntities = findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceTemperatures); + loadedEntities.forEach(entity -> Assert.assertTrue(devices.stream().map(Device::getId).collect(Collectors.toSet()).contains(entity.getEntityId()))); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -616,21 +581,8 @@ public class EntityServiceTest extends AbstractControllerTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = findByQuery(query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -664,18 +616,9 @@ public class EntityServiceTest extends AbstractControllerTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "consumption")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = findByQueryAndCheck(query, 5); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(5, loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); + List deviceTemperatures = consumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "consumption", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -688,21 +631,8 @@ public class EntityServiceTest extends AbstractControllerTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = findByQuery(query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highConsumptions.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highConsumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "consumption", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -1659,89 +1589,48 @@ public class EntityServiceTest extends AbstractControllerTest { List keyFiltersNotEqualTemperature = Collections.singletonList(notEqualTemperatureFilter); //Greater Operation + List deviceTemperatures = greaterTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterTemperature); - PageData data = findByQueryAndCheck(query, greaterTemperatures.size()); - List loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(greaterTemperatures.size(), loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - List deviceTemperatures = greaterTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Greater or equal Operation + deviceTemperatures = greaterOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterOrEqualTemperature); - data = findByQueryAndCheck(query, greaterOrEqualTemperatures.size()); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(greaterOrEqualTemperatures.size(), loadedEntities.size()); - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = greaterOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Less Operation + deviceTemperatures = lessTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessTemperature); - data = findByQueryAndCheck(query, lessTemperatures.size()); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(lessTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = lessTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Less or equal Operation + deviceTemperatures = lessOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessOrEqualTemperature); - data = findByQueryAndCheck(query, lessOrEqualTemperatures.size()); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(lessOrEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = lessOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Equal Operation + deviceTemperatures = equalTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualTemperature); - data = findByQueryAndCheck(query, equalTemperatures.size()); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(equalTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = equalTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Not equal Operation + deviceTemperatures = notEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualTemperature); - data = findByQueryAndCheck(query, notEqualTemperatures.size()); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(notEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = notEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); - + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -1786,24 +1675,10 @@ public class EntityServiceTest extends AbstractControllerTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = findByQueryAndCheck(query, 67); - - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(67, loadedEntities.size()); - List loadedTemperatures = new ArrayList<>(); - for (Device device : devices) { - loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) - .getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); - } List deviceTemperatures = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -1814,22 +1689,10 @@ public class EntityServiceTest extends AbstractControllerTest { highTemperatureFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(highTemperatureFilter); - query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = findByQueryAndCheck(query, highTemperatures.size()); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -2509,13 +2372,14 @@ public class EntityServiceTest extends AbstractControllerTest { return result; } - protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetry) { - List entitiesTelemetry = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry); + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetry) { + List loadedEntities = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry); + List entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList(); assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry); - return entitiesTelemetry; + return loadedEntities; } - protected List findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + protected List findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { PageData data = findByQueryAndCheck(query, expectedTelemetries.size()); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { @@ -2523,7 +2387,7 @@ public class EntityServiceTest extends AbstractControllerTest { data = findByQuery(query); loadedEntities.addAll(data.getData()); } - return loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList(); + return loadedEntities; } protected long countByQuery(CustomerId customerId, EntityCountQuery query) { From 32a4a7ce35627e4fae92739587df37acf2e01bf9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 5 Mar 2025 15:26:58 +0200 Subject: [PATCH 49/51] EDQS: introduce API auto-enable option --- .../service/edqs/DefaultEdqsApiService.java | 11 ++++++++++- .../server/service/edqs/DefaultEdqsService.java | 16 ++++++++++------ application/src/main/resources/thingsboard.yml | 7 +++++-- .../EdqsEntityQueryControllerTest.java | 3 ++- .../service/entitiy/EdqsEntityServiceTest.java | 3 ++- .../test/resources/application-test.properties | 2 +- .../server/common/msg/edqs/EdqsApiService.java | 2 ++ .../dao/sql/query/DummyEdqsApiService.java | 5 +++++ docker/tb-core-edqs.env | 2 +- 9 files changed, 38 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java index d78ea85fa7..51c963ed2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java @@ -22,6 +22,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -43,13 +44,16 @@ import java.util.UUID; @Service @Slf4j @RequiredArgsConstructor -@ConditionalOnExpression("'${queue.edqs.api_enabled:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')") +@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')") public class DefaultEdqsApiService implements EdqsApiService { private final EdqsPartitionService edqsPartitionService; private final EdqsClientQueueFactory queueFactory; private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + @Value("${queue.edqs.api.auto_enable:true}") + private boolean autoEnable; + private Boolean apiEnabled = null; @PostConstruct @@ -100,6 +104,11 @@ public class DefaultEdqsApiService implements EdqsApiService { return true; } + @Override + public boolean isAutoEnable() { + return autoEnable; + } + @PreDestroy private void stop() { requestTemplate.stop(); diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 65f75c1aa4..e823dee4e7 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -118,7 +118,7 @@ public class DefaultEdqsService implements EdqsService { .syncRequest(new EdqsSyncRequest()) .build()); } - } else if (edqsApiService.isSupported()) { + } else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) { // only if topic/RocksDB is not empty and sync is finished edqsApiService.setEnabled(true); } @@ -159,11 +159,15 @@ public class DefaultEdqsService implements EdqsService { edqsSyncService.sync(); saveSyncState(EdqsSyncStatus.FINISHED); - if (edqsApiService.isSupported()) { - broadcast(ToCoreEdqsMsg.builder() - .apiEnabled(Boolean.TRUE) - .build()); - } + if (edqsApiService.isSupported()) + if (edqsApiService.isAutoEnable()) { + log.info("EDQS sync is finished, auto-enabling API"); + broadcast(ToCoreEdqsMsg.builder() + .apiEnabled(Boolean.TRUE) + .build()); + } else { + log.info("EDQS sync is finished, but leaving API disabled"); + } } catch (Exception e) { log.error("Failed to complete sync", e); saveSyncState(EdqsSyncStatus.FAILED); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7e4e0708d8..b01a0cf55e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1731,8 +1731,11 @@ queue: entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" # Batch size of timeseries data being synced with EDQS ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" - # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) - api_enabled: "${TB_EDQS_API_ENABLED:false}" + api: + # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) + supported: "${TB_EDQS_API_SUPPORTED:false}" + # Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished + auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}" # Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices) mode: "${TB_EDQS_MODE:local}" local: diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index a3ce628406..ce5221ab89 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -37,7 +37,8 @@ import static org.awaitility.Awaitility.await; // "queue.type=kafka", // uncomment to use Kafka // "queue.kafka.bootstrap.servers=10.7.1.254:9092", "queue.edqs.sync.enabled=true", - "queue.edqs.api_enabled=true", + "queue.edqs.api.supported=true", + "queue.edqs.api.auto_enable=true", "queue.edqs.mode=local" }) public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index 7f29821afd..4d444ad637 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -51,7 +51,8 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { "queue.edqs.sync.enabled=true", - "queue.edqs.api_enabled=true", + "queue.edqs.api.supported=true", + "queue.edqs.api.auto_enable=true", "queue.edqs.mode=local" }) public class EdqsEntityServiceTest extends EntityServiceTest { diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 3d0c377c2e..b8bdcf67e6 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -60,4 +60,4 @@ server.log_controller_error_stack_trace=false transport.gateway.dashboard.sync.enabled=false queue.edqs.sync.enabled=false -queue.edqs.api_enabled=false +queue.edqs.api.supported=false diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java index 58a86823ca..05864fe863 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java @@ -31,4 +31,6 @@ public interface EdqsApiService { boolean isSupported(); + boolean isAutoEnable(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java index 7b3aed253c..e486d3b645 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java @@ -50,4 +50,9 @@ public class DummyEdqsApiService implements EdqsApiService { return false; } + @Override + public boolean isAutoEnable() { + return false; + } + } diff --git a/docker/tb-core-edqs.env b/docker/tb-core-edqs.env index 1417b780ec..45472ec907 100644 --- a/docker/tb-core-edqs.env +++ b/docker/tb-core-edqs.env @@ -2,4 +2,4 @@ TB_EDQS_MODE=remote TB_EDQS_SYNC_ENABLED=true -TB_EDQS_API_ENABLED=true \ No newline at end of file +TB_EDQS_API_SUPPORTED=true From e6428a8e78bb0c043eb3cc8a569ee855ef411b71 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 6 Mar 2025 15:21:37 +0200 Subject: [PATCH 50/51] increased timeout for edqs tests --- .../server/service/entitiy/EdqsEntityServiceTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index 4d444ad637..50c80d08c7 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -104,13 +104,13 @@ public class EdqsEntityServiceTest extends EntityServiceTest { @Override protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { - return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), result -> result.getTotalElements() == expectedResultSize); } @Override protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { - return await().atMost(15, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries)); } @@ -121,7 +121,7 @@ public class EdqsEntityServiceTest extends EntityServiceTest { @Override protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { - return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query), + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query), result -> result == expectedResult); } From 5028b7b551d7113d895f977ed45e25b22f60d864 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 6 Mar 2025 15:43:23 +0200 Subject: [PATCH 51/51] added Lazy for edqsApiService --- .../org/thingsboard/server/dao/entity/BaseEntityService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 6e32901517..a762aabf38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -86,6 +86,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe EntityServiceRegistry entityServiceRegistry; @Autowired + @Lazy private EdqsApiService edqsApiService; @Override