From 0e9d0b3ba3b456b71fd732a7504d714ac0abc58c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 3 Feb 2025 18:27:36 +0200 Subject: [PATCH 1/6] fixed EntityServiceTest --- .../server/service/edqs/EdqsSyncService.java | 95 ++----- .../EdqsEntityQueryControllerTest.java | 10 +- .../controller/EntityQueryControllerTest.java | 36 +-- .../entitiy/EdqsEntityServiceTest.java | 11 +- .../service/entitiy/EntityServiceTest.java | 244 +----------------- .../server/common/data/ObjectType.java | 17 +- .../data/edqs/fields/DashboardFields.java | 32 ++- .../common/data/edqs/fields/EntityFields.java | 6 + .../data/edqs/fields/EntityViewFields.java | 7 + .../common/data/edqs/fields/FieldsUtil.java | 2 - .../common/data/edqs/query/EdqsRequest.java | 4 - .../common/data/edqs/query/QueryResult.java | 4 +- .../common/data/permission/QueryContext.java | 3 + .../server/common/data/queue/QueueStats.java | 8 +- common/edqs/pom.xml | 4 +- .../server/edqs/data/ApiUsageStateData.java | 2 +- .../server/edqs/data/AssetData.java | 2 +- .../server/edqs/data/BaseEntityData.java | 2 +- .../server/edqs/data/CustomerData.java | 2 +- .../server/edqs/data/DeviceData.java | 2 +- .../server/edqs/data/EntityData.java | 2 +- .../server/edqs/data/EntityGroupData.java | 58 ----- .../server/edqs/data/EntityProfileData.java | 2 +- .../server/edqs/data/GenericData.java | 2 +- .../server/edqs/data/ProfileAwareData.java | 2 +- .../server/edqs/data/RelationData.java | 2 +- .../server/edqs/data/RelationInfo.java | 2 +- .../server/edqs/data/RelationsRepo.java | 2 +- .../server/edqs/data/TenantData.java | 2 +- .../edqs/data/dp/AbstractDataPoint.java | 2 +- .../server/edqs/data/dp/BoolDataPoint.java | 2 +- .../edqs/data/dp/CompressedJsonDataPoint.java | 2 +- .../data/dp/CompressedStringDataPoint.java | 2 +- .../server/edqs/data/dp/DataPoint.java | 2 +- .../server/edqs/data/dp/DoubleDataPoint.java | 2 +- .../server/edqs/data/dp/JsonDataPoint.java | 2 +- .../server/edqs/data/dp/LongDataPoint.java | 2 +- .../server/edqs/data/dp/StringDataPoint.java | 2 +- .../server/edqs/load/TenantRepoLoader.java | 2 +- .../server/edqs/processor/EdqsConverter.java | 2 +- .../server/edqs/processor/EdqsProcessor.java | 9 +- .../server/edqs/processor/EdqsProducer.java | 2 +- .../server/edqs/query/DataKey.java | 2 +- .../server/edqs/query/EdqsCountQuery.java | 2 +- .../server/edqs/query/EdqsDataQuery.java | 2 +- .../server/edqs/query/EdqsFilter.java | 2 +- .../server/edqs/query/EdqsQuery.java | 2 +- .../server/edqs/query/SortableEntityData.java | 4 +- .../AbstractEntityGroupQueryProcessor.java | 75 ------ ...stractEntityProfileNameQueryProcessor.java | 2 +- .../AbstractEntityProfileQueryProcessor.java | 2 +- .../AbstractEntitySearchQueryProcessor.java | 4 +- .../processor/AbstractQueryProcessor.java | 71 +---- .../AbstractRelationQueryProcessor.java | 111 +------- .../AbstractSimpleQueryProcessor.java | 53 +--- ...bstractSingleEntityTypeQueryProcessor.java | 116 +-------- .../ApiUsageStateQueryProcessor.java | 35 +-- .../processor/AssetSearchQueryProcessor.java | 2 +- .../processor/AssetTypeQueryProcessor.java | 2 +- .../query/processor/CombinedPermissions.java | 25 -- .../processor/DeviceSearchQueryProcessor.java | 2 +- .../processor/DeviceTypeQueryProcessor.java | 2 +- .../processor/EdgeTypeQueryProcessor.java | 2 +- .../EdgeTypeSearchQueryProcessor.java | 2 +- .../EntitiesByGroupNameQueryProcessor.java | 121 --------- .../EntitiesByGroupQueryProcessor.java | 96 ------- .../EntityGroupListQueryProcessor.java | 84 ------ .../EntityGroupNameQueryProcessor.java | 83 ------ .../processor/EntityListQueryProcessor.java | 34 +-- .../processor/EntityNameQueryProcessor.java | 2 +- .../query/processor/EntityQueryProcessor.java | 2 +- .../EntityQueryProcessorFactory.java | 8 +- .../processor/EntityTypeQueryProcessor.java | 2 +- .../EntityViewSearchQueryProcessor.java | 2 +- .../EntityViewTypeQueryProcessor.java | 2 +- .../query/processor/GroupPermissions.java | 29 --- .../edqs/query/processor/Permissions.java | 24 -- .../processor/RelationQueryPermissions.java | 33 --- .../processor/RelationQueryProcessor.java | 2 +- .../SchedulerEventQueryProcessor.java | 37 --- .../processor/SingleEntityQueryProcessor.java | 36 +-- .../StateEntityOwnerQueryProcessor.java | 91 ------- .../server/edqs/repo/EdqRepository.java | 7 +- .../edqs/repo/InMemoryEdqRepository.java | 22 +- .../server/edqs/repo/KeyDictionary.java | 2 +- .../server/edqs/repo/TbBytePool.java | 2 +- .../server/edqs/repo/TbStringPool.java | 2 +- .../server/edqs/repo/TenantRepo.java | 142 ++-------- .../server/edqs/state/EdqsStateService.java | 2 +- .../edqs/state/KafkaEdqsStateService.java | 2 +- .../edqs/state/LocalEdqsStateService.java | 2 +- .../server/edqs/stats/EdqsStatsService.java | 2 +- .../edqs/util/EdqsPartitionService.java | 2 +- .../server/edqs/util/EdqsRocksDb.java | 2 +- .../server/edqs/util/RepositoryUtils.java | 39 +-- .../server/edqs/util/TbRocksDb.java | 2 +- .../server/edqs/util/VersionsStore.java | 2 +- .../server/queue/discovery/QueueKey.java | 6 + .../java/org/thingsboard/server/dao/Dao.java | 5 +- .../dao/customer/CustomerServiceImpl.java | 2 +- .../server/dao/entity/BaseEntityService.java | 25 +- .../dao/sql/asset/AssetProfileRepository.java | 5 +- .../server/dao/sql/asset/AssetRepository.java | 5 +- .../server/dao/sql/asset/JpaAssetDao.java | 5 +- .../dao/sql/asset/JpaAssetProfileDao.java | 5 +- .../dao/sql/customer/CustomerRepository.java | 9 +- .../dao/sql/customer/JpaCustomerDao.java | 6 +- .../sql/dashboard/DashboardRepository.java | 6 +- .../dao/sql/dashboard/JpaDashboardDao.java | 5 +- .../sql/device/DeviceProfileRepository.java | 6 +- .../dao/sql/device/DeviceRepository.java | 5 +- .../server/dao/sql/device/JpaDeviceDao.java | 5 +- .../dao/sql/device/JpaDeviceProfileDao.java | 5 +- .../server/dao/sql/edge/EdgeRepository.java | 6 +- .../server/dao/sql/edge/JpaEdgeDao.java | 5 +- .../sql/entityview/EntityViewRepository.java | 7 +- .../dao/sql/entityview/JpaEntityViewDao.java | 7 +- .../query/DefaultAlarmQueryRepository.java | 3 - .../dao/sql/queue/JpaQueueStatsDao.java | 5 +- .../dao/sql/queue/QueueStatsRepository.java | 5 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 6 +- .../dao/sql/rule/RuleChainRepository.java | 5 +- .../server/dao/sql/tenant/JpaTenantDao.java | 11 +- .../dao/sql/tenant/JpaTenantProfileDao.java | 5 +- .../sql/tenant/TenantProfileRepository.java | 5 +- .../dao/sql/tenant/TenantRepository.java | 5 +- .../usagerecord/ApiUsageStateRepository.java | 8 +- .../sql/usagerecord/JpaApiUsageStateDao.java | 6 +- .../server/dao/sql/user/JpaUserDao.java | 5 +- .../server/dao/sql/user/UserRepository.java | 7 +- .../dao/sql/widget/JpaWidgetTypeDao.java | 6 +- .../dao/sql/widget/JpaWidgetsBundleDao.java | 7 +- .../dao/sql/widget/WidgetTypeRepository.java | 6 +- .../sql/widget/WidgetsBundleRepository.java | 6 +- edqs/pom.xml | 5 +- .../edqs/DummyQueueRoutingInfoService.java | 2 +- .../edqs/DummyTenantRoutingInfoService.java | 2 +- .../edqs/ThingsboardEdqsApplication.java | 4 +- edqs/src/main/resources/edqs.yml | 2 +- edqs/src/main/resources/logback.xml | 2 +- .../server/edqs/repo/AbstractEDQTest.java | 51 +--- .../edqs/repo/ApiUsageStateFilterTest.java | 5 +- .../edqs/repo/AssetSearchQueryFilterTest.java | 114 ++------ .../server/edqs/repo/AssetTypeFilterTest.java | 41 ++- .../repo/DeviceSearchQueryFilterTest.java | 133 ++-------- .../edqs/repo/DeviceTypeFilterTest.java | 15 +- .../edqs/repo/EdgeSearchQueryFilterTest.java | 81 ++---- .../server/edqs/repo/EdgeTypeFilterTest.java | 25 +- .../repo/EntitiesByGroupIdFilterTest.java | 161 ------------ .../repo/EntitiesByGroupNameFilterTest.java | 140 ---------- .../edqs/repo/EntityGroupListFilterTest.java | 189 -------------- .../edqs/repo/EntityGroupNameFilterTest.java | 186 ------------- .../edqs/repo/EntityListFilterTest.java | 71 +---- .../edqs/repo/EntityNameFilterTest.java | 15 +- .../edqs/repo/EntityTypeFilterTest.java | 15 +- .../repo/EntityViewSearchQueryFilterTest.java | 131 ++-------- .../edqs/repo/EntityViewTypeFilterTest.java | 25 +- .../edqs/repo/RelationsQueryFilterTest.java | 91 +------ .../server/edqs/repo/RepositoryUtilsTest.java | 2 +- .../edqs/repo/SchedulerEventFilterTest.java | 129 --------- .../edqs/repo/SingleEntityFilterTest.java | 55 +--- .../edqs/repo/StateEntityOwnerFilterTest.java | 81 ------ pom.xml | 1 + 163 files changed, 557 insertions(+), 3488 deletions(-) delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java delete mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java delete mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index bf8e096059..04633892e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.edqs.Entity; import org.thingsboard.server.common.data.edqs.LatestTsKv; import org.thingsboard.server.common.data.edqs.fields.EntityFields; -import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -37,7 +36,6 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.entity.EntityDaoRegistry; -import org.thingsboard.server.dao.group.EntityGroupDao; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -46,39 +44,16 @@ import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; import org.thingsboard.server.dao.tenant.TenantDao; -import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.common.data.ObjectType.API_USAGE_STATE; -import static org.thingsboard.server.common.data.ObjectType.ASSET; -import static org.thingsboard.server.common.data.ObjectType.ASSET_PROFILE; import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; -import static org.thingsboard.server.common.data.ObjectType.BLOB_ENTITY; -import static org.thingsboard.server.common.data.ObjectType.CONVERTER; -import static org.thingsboard.server.common.data.ObjectType.CUSTOMER; -import static org.thingsboard.server.common.data.ObjectType.DASHBOARD; -import static org.thingsboard.server.common.data.ObjectType.DEVICE; -import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE; -import static org.thingsboard.server.common.data.ObjectType.EDGE; -import static org.thingsboard.server.common.data.ObjectType.ENTITY_GROUP; -import static org.thingsboard.server.common.data.ObjectType.ENTITY_VIEW; -import static org.thingsboard.server.common.data.ObjectType.INTEGRATION; import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; -import static org.thingsboard.server.common.data.ObjectType.QUEUE_STATS; import static org.thingsboard.server.common.data.ObjectType.RELATION; -import static org.thingsboard.server.common.data.ObjectType.ROLE; -import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN; -import static org.thingsboard.server.common.data.ObjectType.SCHEDULER_EVENT; -import static org.thingsboard.server.common.data.ObjectType.TENANT; -import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE; -import static org.thingsboard.server.common.data.ObjectType.USER; -import static org.thingsboard.server.common.data.ObjectType.WIDGETS_BUNDLE; -import static org.thingsboard.server.common.data.ObjectType.WIDGET_TYPE; +import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; @Slf4j public abstract class EdqsSyncService { @@ -94,8 +69,6 @@ public abstract class EdqsSyncService { @Autowired private RelationRepository relationRepository; @Autowired - private EntityGroupDao entityGroupDao; - @Autowired private TsKvLatestRepository tsKvLatestRepository; @Autowired @Lazy @@ -106,12 +79,6 @@ public abstract class EdqsSyncService { private final Map counters = new ConcurrentHashMap<>(); - public static final Set edqsTenantTypes = EnumSet.of( - TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, - RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, - BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS - ); - public abstract boolean isSyncNeeded(); public void sync() { @@ -119,9 +86,7 @@ public abstract class EdqsSyncService { long startTs = System.currentTimeMillis(); counters.clear(); - syncTenants(); syncTenantEntities(); - syncEntityGroups(); syncRelations(); loadKeyDictionary(); syncAttributes(); @@ -139,49 +104,28 @@ public abstract class EdqsSyncService { edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); } - private void syncTenants() { - log.info("Synchronizing tenants to EDQS"); - long ts = System.currentTimeMillis(); - var tenants = new PageDataIterable<>(tenantDao::findAllFields, 10000); - for (EntityFields entityFields : tenants) { - TenantId tenantId = TenantId.fromUUID(entityFields.getId()); - entityInfoMap.put(entityFields.getId(), new EntityIdInfo(EntityType.TENANT, tenantId)); - process(tenantId, TENANT, new Entity(EntityType.TENANT, entityFields)); - } - process(TenantId.SYS_TENANT_ID, TENANT, new Entity(EntityType.TENANT, new TenantFields(TenantId.SYS_TENANT_ID.getId(), Long.MAX_VALUE))); - log.info("Finished synchronizing tenants to EDQS in {} ms", (System.currentTimeMillis() - ts)); - } - private void syncTenantEntities() { for (ObjectType type : edqsTenantTypes) { - log.info("Synchronizing tenant {} entities to EDQS", type); + log.info("Synchronizing {} entities to EDQS", type); long ts = System.currentTimeMillis(); EntityType entityType = type.toEntityType(); Dao dao = entityDaoRegistry.getDao(entityType); - var entities = new PageDataIterable<>(dao::findAllFields, 10000); - for (EntityFields entityFields : entities) { - TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); - entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); - process(tenantId, type, new Entity(type.toEntityType(), entityFields)); - } - log.info("Finished synchronizing tenant {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); - } - } - - private void syncEntityGroups() { - log.info("Synchronizing entity groups to EDQS"); - long ts = System.currentTimeMillis(); - var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 30000); - for (EntityFields groupFields : entityGroups) { - EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId()); - if (entityIdInfo != null) { - entityInfoMap.put(groupFields.getId(), new EntityIdInfo(EntityType.ENTITY_GROUP, entityIdInfo.tenantId())); - process(entityIdInfo.tenantId(), ENTITY_GROUP, new Entity(EntityType.ENTITY_GROUP, groupFields)); - } else { - log.info("Entity group owner not found: " + groupFields.getOwnerId()); + UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + while (true) { + var batch = dao.findNextBatch(lastFromEntityId, 10000); + if (batch.isEmpty()) { + break; + } + for (EntityFields entityFields : batch) { + TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); + process(tenantId, type, new Entity(entityType, entityFields)); + } + EntityFields lastRecord = batch.get(batch.size() - 1); + lastFromEntityId = lastRecord.getId(); } + log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); } - log.info("Finished synchronizing entity groups to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private void syncRelations() { @@ -215,7 +159,7 @@ public abstract class EdqsSyncService { private void processRelationBatch(List relations) { for (RelationEntity relation : relations) { - if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup()) || (RelationTypeGroup.FROM_ENTITY_GROUP.name().equals(relation.getRelationTypeGroup()))) { + if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) { EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); if (entityIdInfo != null) { process(entityIdInfo.tenantId(), RELATION, relation.toData()); @@ -284,7 +228,7 @@ public abstract class EdqsSyncService { int lastKey = Integer.MIN_VALUE; while (true) { - List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); if (batch.isEmpty()) { break; } @@ -332,6 +276,7 @@ public abstract class EdqsSyncService { return strKey; } - public record EntityIdInfo(EntityType entityType, TenantId tenantId) {} + public record EntityIdInfo(EntityType entityType, TenantId tenantId) { + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index daa1c92c1c..a75461faaa 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -28,13 +28,14 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; import java.util.concurrent.TimeUnit; +import java.util.function.BiPredicate; import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { - "queue.type=kafka", // uncomment to use Kafka - "queue.kafka.bootstrap.servers=10.7.1.254:9092", +// "queue.type=kafka", // uncomment to use Kafka +// "queue.kafka.bootstrap.servers=10.7.1.254:9092", "queue.edqs.sync_enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" @@ -64,4 +65,9 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { result -> result == expectedResult); } + @Override + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> condition.test(result, expectedResult)); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index ad554f11ba..3941f16b88 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -73,6 +73,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.BiPredicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @@ -80,7 +81,7 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest -public class EntityQueryControllerTest extends AbstractControllerTest { +public abstract class EntityQueryControllerTest extends AbstractControllerTest { private static final String CUSTOMER_USER_EMAIL = "entityQueryCustomer@thingsboard.org"; private static final String TENANT_PASSWORD = "testPassword1"; @@ -157,17 +158,10 @@ public class EntityQueryControllerTest extends AbstractControllerTest { @Test public void testSysAdminCountEntitiesByQuery() throws Exception { - loginSysAdmin(); - - EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); - allDeviceFilter.setEntityType(EntityType.DEVICE); - EntityCountQuery query = new EntityCountQuery(allDeviceFilter); - Long initialCount = countByQuery(query); - loginTenantAdmin(); List devices = new ArrayList<>(); - String devicePrefix = "Device" + RandomStringUtils.random(5); + String devicePrefix = "Device" + RandomStringUtils.randomAlphabetic(5); for (int i = 0; i < 97; i++) { Device device = new Device(); device.setName(devicePrefix + i); @@ -177,18 +171,18 @@ public class EntityQueryControllerTest extends AbstractControllerTest { Thread.sleep(1); } DeviceTypeFilter filter = new DeviceTypeFilter(); - filter.setDeviceType("default"); + filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter(""); loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); - countByQueryAndCheck(countQuery, initialCount + 97); + countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); - filter.setDeviceType("unknown"); + filter.setDeviceTypes(List.of("unknown")); countByQueryAndCheck(countQuery, 0); - filter.setDeviceType("default"); + filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter(devicePrefix + "1"); countByQueryAndCheck(countQuery, 11); @@ -199,7 +193,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { countQuery = new EntityCountQuery(entityListFilter); countByQueryAndCheck(countQuery, 97); - countByQueryAndCheck(query, initialCount + 97); + countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); } @Test @@ -847,9 +841,9 @@ public class EntityQueryControllerTest extends AbstractControllerTest { var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == expectedNumOfDevices; }); - if (expectedNumOfDevices == 0) { - return; - } + if (expectedNumOfDevices == 0) { + return; + } var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); @@ -870,7 +864,8 @@ public class EntityQueryControllerTest extends AbstractControllerTest { } protected PageData findByQuery(EntityDataQuery query) throws Exception { - return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); + return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() { + }); } protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception { @@ -911,4 +906,9 @@ public class EntityQueryControllerTest extends AbstractControllerTest { return numericFilter; } + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) throws Exception { + Long result = countByQuery(query); + assertThat(condition.test(result, expectedResult)).isTrue(); + return result; + } } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index eea7e0eb7f..cf24deb7e4 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -21,7 +21,6 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.permission.MergedUserPermissions; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -53,19 +52,19 @@ public class EdqsEntityServiceTest extends EntityServiceTest { } @Override - protected PageData findByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityDataQuery query, long expectedResultSize) { - return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, permissions, query), + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), result -> result.getTotalElements() == expectedResultSize); } @Override protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { - return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), mergedUserPermissionsPE, countQuery, expectedResult); + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); } @Override - protected long countByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityCountQuery query, int expectedResult) { - return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, permissions, query), + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query), result -> result == expectedResult); } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 04bad95be7..fab4cba3f5 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -489,7 +489,7 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - countByQueryAndCheck(countQuery, 9); + countByQueryAndCheck(countQuery, 3); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); @@ -1444,30 +1444,16 @@ public class EntityServiceTest extends AbstractControllerTest { assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); // find by customer user with generic permission - MergedUserPermissions mergedGenericPermission = new MergedUserPermissions(Map.of(Resource.DEVICE, Set.of(Operation.READ)), Collections.emptyMap()); - PageData customerResults = findByQueryAndCheck(customerId, mergedGenericPermission, query, 1); + PageData customerResults = findByQueryAndCheck(customerId, query, 1); String cutomerDeviceName = customerResults.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(cutomerDeviceName).isEqualTo(customerDevices.get(0).getName()); - // find by customer user with group permission - MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(customerDeviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); - PageData result2 = findByQueryAndCheck(customerId, mergedGroupOnlyPermission, query, 1); - - String resultDeviceName2 = result2.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(resultDeviceName2).isEqualTo(customerDevices.get(0).getName()); - // try to find tenant device by customer user SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); - findByQueryAndCheck(customerId, mergedGenericPermission, customerQuery2, 0); - - // find by tenant user with group permission - PageData results3 = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), mergedGroupOnlyPermission, query, 1); - - String deviceName3 = results3.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(deviceName3).isEqualTo(customerDevices.get(0).getName()); + findByQueryAndCheck(customerId, customerQuery2, 0); } private List getResultDeviceIds(PageData result) { @@ -1634,230 +1620,6 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } - @Test - public void testFindEntityDataByRelationQuery_blobEntity_customerLevel() { - final int deviceCnt = 2; - final int relationsCnt = 3; - final int blobEntitiesCnt = deviceCnt * relationsCnt; - - Customer customer = new Customer(); - customer.setTenantId(tenantId); - customer.setTitle("Customer Relation Query"); - customer = customerService.saveCustomer(customer); - - List devices = new ArrayList<>(); - for (int i = 0; i < deviceCnt; i++) { - Device device = new Device(); - device.setTenantId(tenantId); - device.setName("Device relation query " + i); - device.setCustomerId(customer.getId()); - device.setType("default"); - devices.add(deviceService.saveDevice(device)); - } - - List blobEntities = new ArrayList<>(); - for (int i = 0; i < blobEntitiesCnt; i++) { - BlobEntity blobEntity = new BlobEntity(); - blobEntity.setName("Blob relation query " + i); - blobEntity.setTenantId(tenantId); - blobEntity.setContentType("image/png"); - blobEntity.setData(ByteBuffer.allocate(1024)); - blobEntity.setCustomerId(customer.getId()); - blobEntity.setType("Report"); - blobEntities.add(blobEntityService.saveBlobEntity(blobEntity)); - } - - for (int i = 0; i < deviceCnt; i++) { - for (int j = 0; j < relationsCnt; j++) { - EntityRelation relationEntity = new EntityRelation(); - relationEntity.setFrom(devices.get(i).getId()); - relationEntity.setTo(blobEntities.get(j + (i * relationsCnt)).getId()); - relationEntity.setTypeGroup(RelationTypeGroup.COMMON); - relationEntity.setType("fileAttached"); - relationService.saveRelation(tenantId, relationEntity); - } - } - - MergedUserPermissions mergedUserPermissions = new MergedUserPermissions(Map.of(ALL, Set.of(Operation.ALL)), Collections.emptyMap()); - - RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("fileAttached", Collections.singletonList(EntityType.BLOB_ENTITY)); - RelationsQueryFilter filter = new RelationsQueryFilter(); - filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); - filter.setDirection(EntitySearchDirection.FROM); - EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); - - for (Device device : devices) { - filter.setRootEntity(device.getId()); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); - findByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); - countByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); - /* - In order to be careful with updating Relation Query while adding new Entity Type, - this checkup will help to find place, where you could check the correctness of building query - */ - Assert.assertEquals(38, EntityType.values().length); - } - } - - @Test - public void testFindEntitiesByRelationEntityTypeFilterWithTenantGroupPermission() { - final int assetCount = 2; - final int relationsCnt = 4; - final int deviceEntitiesCnt = assetCount * relationsCnt; - - EntityGroup deviceGroup = new EntityGroup(); - deviceGroup.setName("Device Tenant Level Group"); - deviceGroup.setOwnerId(tenantId); - deviceGroup.setTenantId(tenantId); - deviceGroup.setType(EntityType.DEVICE); - deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); - - List assets = new ArrayList<>(); - for (int i = 0; i < assetCount; i++) { - Asset building = new Asset(); - building.setTenantId(tenantId); - building.setName("Building _" + i); - building.setType("building"); - building = assetService.saveAsset(building); - assets.add(building); - } - - List devices = new ArrayList<>(); - for (int i = 0; i < deviceEntitiesCnt; i++) { - Device device = new Device(); - device.setTenantId(tenantId); - device.setName("Test device " + i); - device.setType("default"); - Device savedDevice = deviceService.saveDevice(device); - devices.add(savedDevice); - if (i % 2 == 0) { - entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); - } - } - - for (int i = 0; i < assetCount; i++) { - for (int j = 0; j < relationsCnt; j++) { - EntityRelation relationEntity = new EntityRelation(); - relationEntity.setFrom(assets.get(i).getId()); - relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); - relationEntity.setTypeGroup(RelationTypeGroup.COMMON); - relationEntity.setType("contains"); - relationService.saveRelation(tenantId, relationEntity); - } - } - - MergedUserPermissions groupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), - Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); - - RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); - RelationsQueryFilter filter = new RelationsQueryFilter(); - filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); - filter.setDirection(EntitySearchDirection.FROM); - EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); - List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); - - for (Asset asset : assets) { - filter.setRootEntity(asset.getId()); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); - countByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); - } - } - - @Test - public void testFindEntitiesWithRelationEntityTypeFilterByCustomerUser() { - Customer customer = new Customer(); - customer.setTenantId(tenantId); - customer.setTitle("Customer Relation Query"); - customer = customerService.saveCustomer(customer); - - final int assetCount = 2; - final int relationsCnt = 4; - final int deviceEntitiesCnt = assetCount * relationsCnt; - - EntityGroup deviceGroup = new EntityGroup(); - deviceGroup.setName("Device Tenant Level Group"); - deviceGroup.setOwnerId(customer.getId()); - deviceGroup.setTenantId(tenantId); - deviceGroup.setType(EntityType.DEVICE); - deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); - - List assets = new ArrayList<>(); - for (int i = 0; i < assetCount; i++) { - Asset building = new Asset(); - building.setTenantId(tenantId); - building.setCustomerId(customer.getId()); - building.setName("Building _" + i); - building.setType("building"); - building = assetService.saveAsset(building); - assets.add(building); - } - - List devices = new ArrayList<>(); - for (int i = 0; i < deviceEntitiesCnt; i++) { - Device device = new Device(); - device.setTenantId(tenantId); - device.setCustomerId(customer.getId()); - device.setName("Test device " + i); - device.setType("default"); - Device savedDevice = deviceService.saveDevice(device); - devices.add(savedDevice); - if (i % 2 == 0) { - entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); - } - } - - for (int i = 0; i < assetCount; i++) { - for (int j = 0; j < relationsCnt; j++) { - EntityRelation relationEntity = new EntityRelation(); - relationEntity.setFrom(assets.get(i).getId()); - relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); - relationEntity.setTypeGroup(RelationTypeGroup.COMMON); - relationEntity.setType("contains"); - relationService.saveRelation(tenantId, relationEntity); - } - } - - MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - MergedUserPermissions mergedGenericOnlyPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.emptyMap()); - MergedUserPermissions mergedGenericAndGroupPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); - RelationsQueryFilter filter = new RelationsQueryFilter(); - filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); - filter.setDirection(EntitySearchDirection.FROM); - EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); - List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - - for (Asset asset : assets) { - filter.setRootEntity(asset.getId()); - - //check by user with generic permission - PageData relationsResult = findByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); - countByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); - - //check by user with generic and group permission - PageData relationsResult1 = findByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); - countByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); - - //check by other customer user with group only permission - PageData relationsResult2 = findByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); - long relationsResultCnt2 = countByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); - - Assert.assertEquals(relationsCnt / 2, relationsResult2.getData().size()); - Assert.assertEquals(relationsCnt / 2, relationsResultCnt2); - - //check by other customer user with generic and group only permission - PageData relationsResult3 = findByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); - long relationsResultCnt3 = countByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); - - Assert.assertEquals(relationsCnt / 2, relationsResult3.getData().size()); - Assert.assertEquals(relationsCnt / 2, relationsResultCnt3); - } - } @Test public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index c78998cd6a..d3aa547bf0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -30,16 +30,8 @@ public enum ObjectType { RULE_CHAIN, OTA_PACKAGE, RESOURCE, - ROLE, - ENTITY_GROUP, - DEVICE_GROUP_OTA_PACKAGE, - GROUP_PERMISSION, - BLOB_ENTITY, - SCHEDULER_EVENT, EVENT, RULE_NODE, - CONVERTER, - INTEGRATION, USER, USER_CREDENTIALS, USER_AUTH_SETTINGS, @@ -62,8 +54,6 @@ public enum ObjectType { NOTIFICATION_TARGET, NOTIFICATION_TEMPLATE, NOTIFICATION_RULE, - WHITE_LABELING, - CUSTOM_TRANSLATION, ALARM_COMMENT, ALARM_TYPE, API_USAGE_STATE, @@ -75,16 +65,15 @@ public enum ObjectType { LATEST_TS_KV; public static final Set edqsTenantTypes = EnumSet.of( - TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, - RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, - BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS + TENANT, TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS ); public static final Set edqsTypes = new HashSet<>(edqsTenantTypes); public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); static { - edqsTypes.addAll(Arrays.asList(TENANT, ENTITY_GROUP, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + edqsTypes.addAll(Arrays.asList(TENANT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); } public EntityType toEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java index af1640b7be..fa57f9d0f4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java @@ -15,10 +15,15 @@ */ package org.thingsboard.server.common.data.edqs.fields; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; @@ -27,7 +32,30 @@ import java.util.UUID; @SuperBuilder public class DashboardFields extends AbstractEntityFields { - public DashboardFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { - super(id, createdTime, tenantId, customerId, name, version); + private static ObjectMapper objectMapper = new ObjectMapper(); + private List assignedCustomerIds; + + public DashboardFields(UUID id, long createdTime, UUID tenantId, String assignedCustomers, String name, Long version) { + super(id, createdTime, tenantId, name, version); + this.assignedCustomerIds = getCustomerIds(assignedCustomers); + } + + private static List getCustomerIds(String assignedCustomers) { + List ids = new ArrayList<>(); + if (assignedCustomers == null || assignedCustomers.isEmpty()) { + return ids; + } + try { + JsonNode rootNode = objectMapper.readTree(assignedCustomers); + for (JsonNode node : rootNode) { + String idStr = node.path("customerId").path("id").asText(); + if (!idStr.isEmpty()) { + ids.add(UUID.fromString(idStr)); + } + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return ids; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java index 7536586f57..ba9f3b0e9b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -19,6 +19,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.thingsboard.server.common.data.id.EntityId; +import java.util.Collections; +import java.util.List; import java.util.UUID; public interface EntityFields { @@ -37,6 +39,10 @@ public interface EntityFields { return null; } + default List getAssignedCustomerIds() { + return Collections.emptyList(); + } + default long getCreatedTime() { return 0; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java index 5635566cc4..01c32f820e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java @@ -19,6 +19,8 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; +import java.util.UUID; + @Data @NoArgsConstructor @SuperBuilder @@ -27,4 +29,9 @@ public class EntityViewFields extends AbstractEntityFields { private String type; private String additionalInfo; + public EntityViewFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, String type, String additionalInfo, Long version) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.additionalInfo = additionalInfo; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index f1cf51bf8b..6543a6d779 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -85,7 +85,6 @@ public class FieldsUtil { return CustomerFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) - .customerId(getCustomerId(entity.getCustomerId())) .name(entity.getTitle()) .additionalInfo(getText(entity.getAdditionalInfo())) .email(entity.getEmail()) @@ -199,7 +198,6 @@ public class FieldsUtil { return DashboardFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) - .customerId(getCustomerId(entity.getCustomerId())) .name(entity.getTitle()) .version(entity.getVersion()) .build(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java index 135061a842..b776d18551 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.common.data.edqs.query; -import com.fasterxml.jackson.annotation.JsonIncludeProperties; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.permission.MergedUserPermissions; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -32,7 +30,5 @@ public class EdqsRequest { private EntityDataQuery entityDataQuery; private EntityCountQuery entityCountQuery; - @JsonIncludeProperties({"genericPermissions", "groupPermissions"}) - private MergedUserPermissions userPermissions; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java index 0c45812690..70e25ba14e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java @@ -30,12 +30,10 @@ import java.util.Map; public class QueryResult { private final EntityId entityId; - private final boolean readAttrs; - private final boolean readTs; private final Map> latest; public EntityData toOldEntityData() { - return new EntityData(entityId, readAttrs, readTs, latest, Collections.emptyMap(), Collections.emptyMap()); + return new EntityData(entityId, latest, Collections.emptyMap(), Collections.emptyMap()); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java index b377d63ee4..b8b3f51aec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java @@ -44,4 +44,7 @@ public class QueryContext { this(tenantId, customerId, entityType, false); } + public boolean isTenantUser() { + return customerId == null || customerId.isNullUid(); + } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java index 9c3536dea1..6c648daf02 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java @@ -18,15 +18,13 @@ package org.thingsboard.server.common.data.queue; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasEntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; @EqualsAndHashCode(callSuper = true) @Data -public class QueueStats extends BaseData implements HasTenantId, HasEntityType { +public class QueueStats extends BaseData implements HasTenantId { private TenantId tenantId; private String queueName; private String serviceId; @@ -38,8 +36,4 @@ public class QueueStats extends BaseData implements HasTenantId, H super(id); } - @Override - public EntityType getEntityType() { - return EntityType.QUEUE_STATS; - } } diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml index 40e89acee6..e58c7c97a0 100644 --- a/common/edqs/pom.xml +++ b/common/edqs/pom.xml @@ -1,6 +1,6 @@ A2, A1 --Contains--> D1. A1 --Manages--> D2. - createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); - - MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), - new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); - - PageData relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(1, relationsResult.getData().size()); - Assert.assertTrue(checkContains(relationsResult, da1)); - - relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(0, relationsResult.getData().size()); - } - @Test public void testFindCustomerDevices() { UUID ta1 = createAsset("T A1"); @@ -169,55 +136,15 @@ public class RelationsQueryFilterTest extends AbstractEDQTest { Assert.assertEquals(0, relationsResult.getData().size()); } - @Test - public void testFindCustomerDevicesGroupsOnly() { - UUID ta1 = createAsset("T A1"); - UUID ta2 = createAsset("T A2"); - UUID da1 = createDevice(customerId, "T D1"); - UUID da2 = createDevice(customerId, "T D2"); - UUID da3 = createDevice(customerId, "T D3"); - - UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); - createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - - // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. - createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); - createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); - createRelation(EntityType.DEVICE, da2, EntityType.DEVICE, da3, "Contains"); - - MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), - new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); - - PageData relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(2, relationsResult.getData().size()); - Assert.assertTrue(checkContains(relationsResult, da1)); - Assert.assertTrue(checkContains(relationsResult, da3)); - - relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); - Assert.assertEquals(1, relationsResult.getData().size()); - Assert.assertTrue(checkContains(relationsResult, da2)); - } - private PageData filter(EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(RepositoryUtils.ALL_READ_PERMISSIONS, rootId, relationEntityTypeFilters); - } - - private PageData filter(MergedUserPermissions permissions, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(permissions, null, rootId, relationEntityTypeFilters); + return filter(null, rootId, relationEntityTypeFilters); } private PageData filter(CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, rootId, relationEntityTypeFilters); - } - - private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { - return filter(permissions, customerId, rootId, 3, false, relationEntityTypeFilters); + return filter(customerId, rootId, 3, false, relationEntityTypeFilters); } - private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { + private PageData filter(CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { RelationsQueryFilter filter = new RelationsQueryFilter(); filter.setRootEntity(rootId); filter.setFilters(Arrays.asList(relationEntityTypeFilters)); @@ -227,7 +154,7 @@ public class RelationsQueryFilterTest extends AbstractEDQTest { EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "T"); EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); } } diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java index 42706a796a..48a72a9cc6 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2024 ThingsBoard, Inc. + * Copyright © 2016-2024 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java deleted file mode 100644 index 8ebbff534d..0000000000 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Copyright © 2016-2024 ThingsBoard, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.edqs.repo; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityKeyValueType; -import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.query.KeyFilter; -import org.thingsboard.server.common.data.query.SchedulerEventFilter; -import org.thingsboard.server.common.data.query.StringFilterPredicate; -import org.thingsboard.server.edqs.util.RepositoryUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.UUID; - -public class SchedulerEventFilterTest extends AbstractEDQTest { - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - @Test - public void testFindTenantSchedulerEvents() { - UUID dashboardId = createDashboard("test dashboard"); - UUID deviceId = createDevice("test device"); - - UUID eventId1 = createSchedulerEvent("Update attributes", new DeviceId(deviceId), "Turn off device"); - UUID eventId2 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate morning report"); - UUID eventId3 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate evening report"); - - // find all scheduler events with type "Generate report" - var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); - Assert.assertEquals(2, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - Assert.assertTrue(checkContains(result, eventId3)); - - // find all scheduler events for device originator - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId1)); - - // find all scheduler events with name "%morning%" - KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - } - - @Test - public void testFindCustomerEdges() { - UUID dashboardId = createDashboard( "test dashboard"); - UUID deviceId = createDevice("test device"); - - UUID eventId1 = createSchedulerEvent(customerId.getId(), "Update attributes", new DeviceId(deviceId), "Turn off device"); - UUID eventId2 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate morning report"); - UUID eventId3 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate evening report"); - - // find all scheduler events with type "Generate report" - var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); - Assert.assertEquals(2, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - Assert.assertTrue(checkContains(result, eventId3)); - - // find all scheduler events for device originator - result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId1)); - - // find all scheduler events with name "%morning%" - KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); - result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); - Assert.assertEquals(1, result.getTotalElements()); - Assert.assertTrue(checkContains(result, eventId2)); - } - - private static EntityDataQuery getSchedulerEventQuery(String eventType, EntityId entityId, List keyFilters) { - SchedulerEventFilter filter = new SchedulerEventFilter(); - filter.setEventType(eventType); - filter.setOriginator(entityId); - var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); - - var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); - var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); - - return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - } - - private static KeyFilter getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { - KeyFilter nameFilter = new KeyFilter(); - nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); - var predicate = new StringFilterPredicate(); - predicate.setIgnoreCase(ignoreCase); - predicate.setOperation(operation); - predicate.setValue(new FilterPredicateValue<>(predicateValue)); - nameFilter.setPredicate(predicate); - nameFilter.setValueType(EntityKeyValueType.STRING); - return nameFilter; - } - -} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java index 8755191e39..c7de17e33c 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2024 ThingsBoard, Inc. + * Copyright © 2016-2024 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,16 +22,10 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edqs.LatestTsKv; -import org.thingsboard.server.common.data.edqs.query.QueryResult; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EntityGroupId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; -import org.thingsboard.server.common.data.permission.MergedUserPermissions; -import org.thingsboard.server.common.data.permission.Operation; -import org.thingsboard.server.common.data.permission.Resource; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; @@ -42,12 +36,8 @@ import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.edqs.util.RepositoryUtils; import java.util.Arrays; -import java.util.Map; -import java.util.Set; import java.util.UUID; public class SingleEntityFilterTest extends AbstractEDQTest { @@ -72,7 +62,7 @@ public class SingleEntityFilterTest extends AbstractEDQTest { addOrUpdate(EntityType.DEVICE, device); addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); - var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + var result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); Assert.assertEquals(1, result.getTotalElements()); var first = result.getData().get(0); @@ -80,13 +70,13 @@ public class SingleEntityFilterTest extends AbstractEDQTest { Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); Assert.assertEquals(0, result.getTotalElements()); device.setCustomerId(customerId); addOrUpdate(EntityType.DEVICE, device); - result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); Assert.assertEquals(1, result.getTotalElements()); first = result.getData().get(0); Assert.assertEquals(deviceId, first.getEntityId()); @@ -94,39 +84,6 @@ public class SingleEntityFilterTest extends AbstractEDQTest { Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); } - @Test - public void testFindTenantDeviceWithGenericAndGroupPermission() { - UUID deviceId = createDevice(customerId, "LoRa-customer-1"); - UUID deviceId2 = createDevice(customerId, "LoRa-customer-2"); - UUID deviceId3 = createDevice(customerId, "LoRa-customer-3"); - - // add device and device 2 to Group A - UUID groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); - createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - - // add device and device 2 to Group A - UUID groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); - createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); - - MergedUserPermissions genericAndGroupAPermission = new MergedUserPermissions( - Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - var result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupAPermission, getEntityDataQuery(new DeviceId(deviceId2)), false); - Assert.assertEquals(1, result.getTotalElements()); - QueryResult queryResult = result.getData().get(0); - Assert.assertEquals(deviceId2, queryResult.getEntityId().getId()); - Assert.assertEquals("LoRa-customer-2", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); - - // find device without permission - MergedUserPermissions genericAndGroupBPermission = new MergedUserPermissions( - Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); - result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupBPermission, getEntityDataQuery(new DeviceId(deviceId3)), false); - Assert.assertEquals(1, result.getTotalElements()); - queryResult = result.getData().get(0); - Assert.assertEquals(deviceId3, queryResult.getEntityId().getId()); - Assert.assertEquals("LoRa-customer-3", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); - } - @Test public void testFindCustomerDevice() { DeviceId deviceId = new DeviceId(UUID.randomUUID()); @@ -139,13 +96,13 @@ public class SingleEntityFilterTest extends AbstractEDQTest { addOrUpdate(EntityType.DEVICE, device); addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); - var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); Assert.assertEquals(0, result.getTotalElements()); device.setCustomerId(customerId); addOrUpdate(EntityType.DEVICE, device); - result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); Assert.assertEquals(1, result.getTotalElements()); var first = result.getData().get(0); diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java deleted file mode 100644 index f0bb4c4e0b..0000000000 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright © 2016-2024 ThingsBoard, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.edqs.repo; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityKeyValueType; -import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.query.KeyFilter; -import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; -import org.thingsboard.server.common.data.query.StringFilterPredicate; -import org.thingsboard.server.edqs.util.RepositoryUtils; - -import java.util.Arrays; -import java.util.UUID; - -public class StateEntityOwnerFilterTest extends AbstractEDQTest { - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - @Test - public void testFindCustomerDeviceOwner() { - UUID customerId = UUID.randomUUID(); - createCustomer(customerId, null, "Customer A"); - UUID deviceId = createDevice(new CustomerId(customerId), "LoRa-1"); - - var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(deviceId)), false); - - Assert.assertEquals(1, result.getTotalElements()); - var customer = result.getData().get(0); - Assert.assertEquals(customerId, customer.getEntityId().getId()); - Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); - } - - private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { - StateEntityOwnerFilter filter = new StateEntityOwnerFilter(); - filter.setSingleEntity(deviceId); - var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); - - var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); - KeyFilter nameFilter = new KeyFilter(); - nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); - var predicate = new StringFilterPredicate(); - predicate.setIgnoreCase(false); - predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); - predicate.setValue(new FilterPredicateValue<>("LoRa-")); - nameFilter.setPredicate(predicate); - nameFilter.setValueType(EntityKeyValueType.STRING); - - return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); - } - -} diff --git a/pom.xml b/pom.xml index 42747bae46..c809651aa2 100755 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ 1.7.0 4.4.0 2.2.14 + 0.6.12 3.12.1 2.0.0-M15 2.10.1 From 60736e12377d2f6411e873c2e9f7f827f71b8c42 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 4 Feb 2025 12:23:35 +0200 Subject: [PATCH 2/6] fixed entity removal from CustomerData --- .../server/service/edqs/EdqsSyncService.java | 6 ++-- .../EdqsEntityQueryControllerTest.java | 5 --- .../controller/EntityQueryControllerTest.java | 18 +++++----- .../server/common/data/ObjectType.java | 2 +- .../processor/AbstractQueryProcessor.java | 2 +- .../AbstractRelationQueryProcessor.java | 5 ++- .../processor/EntityListQueryProcessor.java | 2 +- .../processor/SingleEntityQueryProcessor.java | 2 +- .../server/edqs/repo/TenantRepo.java | 35 ++++++++++--------- .../dao/queue/BaseQueueStatsService.java | 2 +- .../server/dao/queue/QueueStatsDao.java | 7 ++-- .../dao/sql/queue/JpaQueueStatsDao.java | 8 ++++- 12 files changed, 48 insertions(+), 46 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 04633892e5..c2750ae840 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -110,9 +110,9 @@ public abstract class EdqsSyncService { long ts = System.currentTimeMillis(); EntityType entityType = type.toEntityType(); Dao dao = entityDaoRegistry.getDao(entityType); - UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); while (true) { - var batch = dao.findNextBatch(lastFromEntityId, 10000); + var batch = dao.findNextBatch(lastId, 10000); if (batch.isEmpty()) { break; } @@ -122,7 +122,7 @@ public abstract class EdqsSyncService { process(tenantId, type, new Entity(entityType, entityFields)); } EntityFields lastRecord = batch.get(batch.size() - 1); - lastFromEntityId = lastRecord.getId(); + lastId = lastRecord.getId(); } log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index a75461faaa..baf9f98993 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -65,9 +65,4 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { result -> result == expectedResult); } - @Override - protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) { - return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), - result -> condition.test(result, expectedResult)); - } } diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 3941f16b88..508f66e584 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -81,7 +81,7 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest -public abstract class EntityQueryControllerTest extends AbstractControllerTest { +public class EntityQueryControllerTest extends AbstractControllerTest { private static final String CUSTOMER_USER_EMAIL = "entityQueryCustomer@thingsboard.org"; private static final String TENANT_PASSWORD = "testPassword1"; @@ -158,6 +158,13 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { @Test public void testSysAdminCountEntitiesByQuery() throws Exception { + loginSysAdmin(); + + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery query = new EntityCountQuery(allDeviceFilter); + countByQueryAndCheck(query, 0); + loginTenantAdmin(); List devices = new ArrayList<>(); @@ -177,7 +184,7 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); - countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); countByQueryAndCheck(countQuery, 0); @@ -193,7 +200,7 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { countQuery = new EntityCountQuery(entityListFilter); countByQueryAndCheck(countQuery, 97); - countByQueryAndCheck(countQuery, 97, (actual, expected) -> actual >= expected); + countByQueryAndCheck(countQuery, 97); } @Test @@ -906,9 +913,4 @@ public abstract class EntityQueryControllerTest extends AbstractControllerTest { return numericFilter; } - protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult, BiPredicate condition) throws Exception { - Long result = countByQuery(query); - assertThat(condition.test(result, expectedResult)).isTrue(); - return result; - } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index d3aa547bf0..bc5ec58213 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -73,7 +73,7 @@ public enum ObjectType { API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); static { - edqsTypes.addAll(Arrays.asList(TENANT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + edqsTypes.addAll(Arrays.asList(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); } public EntityType toEntityType() { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java index 29626dad1f..b795af08db 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java @@ -62,7 +62,7 @@ public abstract class AbstractQueryProcessor implements } } - protected static boolean checkCustomer(UUID customerId, EntityData ed) { + protected static boolean checkCustomerId(UUID customerId, EntityData ed) { return customerId.equals(ed.getCustomerId()) || (ed.getEntityType() == EntityType.DASHBOARD && ed.getFields().getAssignedCustomerIds().contains(customerId)); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java index b5eaf26e6e..2a4e72ff0b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -16,7 +16,6 @@ package org.thingsboard.server.edqs.query.processor; import lombok.RequiredArgsConstructor; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; @@ -79,7 +78,7 @@ public abstract class AbstractRelationQueryProcessor ext } else { var customerId = ctx.getCustomerId().getId(); for (EntityData ed : entities) { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { result++; } } @@ -97,7 +96,7 @@ public abstract class AbstractRelationQueryProcessor ext var customerId = ctx.getCustomerId().getId(); List result = new ArrayList<>(); for (EntityData ed : entities) { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { result.add(toSortData(ed)); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java index 36016009bd..b319a4e13a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java @@ -41,7 +41,7 @@ public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProce @Override protected void processCustomerQuery(UUID customerId, Consumer> processor) { processAll(ed -> { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { processor.accept(ed); } }); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java index 19b32b5a93..55464b4529 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java @@ -39,7 +39,7 @@ public class SingleEntityQueryProcessor extends AbstractSingleEntityTypeQueryPro @Override protected void processCustomerQuery(UUID customerId, Consumer> processor) { processAll(ed -> { - if (checkCustomer(customerId, ed)) { + if (checkCustomerId(customerId, ed)) { processor.accept(ed); } }); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 7773c760f0..68e1c42589 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -195,24 +195,20 @@ public class TenantRepo { processFields(fields); entityData.setFields(entity.getFields()); - switch (entity.getType()) { - default -> { - UUID newCustomerId = fields.getCustomerId(); - UUID oldCustomerId = entityData.getCustomerId(); - entityData.setCustomerId(newCustomerId); - if (entityIdMismatch(oldCustomerId, newCustomerId)) { - if (oldCustomerId != null) { - CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); - if (old != null) { - old.remove(entityData); - } - } - if (newCustomerId != null) { - CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); - newData.addOrUpdate(entityData); - } + UUID newCustomerId = fields.getCustomerId(); + UUID oldCustomerId = entityData.getCustomerId(); + entityData.setCustomerId(newCustomerId); + if (entityIdMismatch(oldCustomerId, newCustomerId)) { + if (oldCustomerId != null) { + CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); + if (old != null) { + old.remove(entityData); } } + if (newCustomerId != null) { + CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); + newData.addOrUpdate(entityData); + } } } finally { entityUpdateLock.unlock(); @@ -228,6 +224,13 @@ public class TenantRepo { if (removed != null) { getEntitySet(entityType).remove(removed); edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + UUID customerId = removed.getCustomerId(); + if (customerId != null) { + CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + customerData.remove(removed); + } + } } } finally { entityUpdateLock.unlock(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index efc9dab41b..e3e6edad94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -85,7 +85,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public PageData findByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findByTenantId, tenantId: [{}]", tenantId); Validator.validatePageLink(pageLink); - return queueStatsDao.findByTenantId(tenantId, pageLink); + return queueStatsDao.findAllByTenantId(tenantId, pageLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java index 5466a8afcc..018eb04d67 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java @@ -17,19 +17,16 @@ package org.thingsboard.server.dao.queue; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface QueueStatsDao extends Dao { +public interface QueueStatsDao extends Dao, TenantEntityDao { QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String queueName, String serviceId); - PageData findByTenantId(TenantId tenantId, PageLink pageLink); - void deleteByTenantId(TenantId tenantId); List findByIds(TenantId tenantId, List queueStatsIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index 97fedd693a..41e2de02a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -21,6 +21,7 @@ import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; @@ -62,10 +63,15 @@ public class JpaQueueStatsDao extends JpaAbstractDao findByTenantId(TenantId tenantId, PageLink pageLink) { + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { return DaoUtil.toPageData(queueStatsRepository.findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } + @Override + public ObjectType getType() { + return ObjectType.QUEUE_STATS; + } + @Override public void deleteByTenantId(TenantId tenantId) { queueStatsRepository.deleteByTenantId(tenantId.getId()); From b2bf5b9f6535bc55c233d4ba7ba13edb156bc08b Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 12:20:23 +0200 Subject: [PATCH 3/6] configurable edqs sync batch sizes --- .../server/service/edqs/DefaultEdqsService.java | 2 +- .../server/service/edqs/EdqsListener.java | 2 +- .../server/service/edqs/EdqsSyncService.java | 16 +++++++++------- .../service/edqs/KafkaEdqsSyncService.java | 2 +- .../service/edqs/LocalEdqsSyncService.java | 2 +- application/src/main/resources/thingsboard.yml | 5 ++++- .../EdqsEntityQueryControllerTest.java | 2 +- .../service/entitiy/EdqsEntityServiceTest.java | 2 +- .../test/resources/application-test.properties | 2 +- .../server/queue/edqs/EdqsComponent.java | 2 +- .../server/queue/edqs/InMemoryEdqsComponent.java | 2 +- .../server/queue/edqs/KafkaEdqsComponent.java | 2 +- .../server/dao/sql/query/DummyEdqsService.java | 2 +- 13 files changed, 24 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 160d4bb565..e7243ed3bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -81,7 +81,7 @@ import java.util.concurrent.TimeUnit; @Service @RequiredArgsConstructor @Slf4j -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") public class DefaultEdqsService implements EdqsService { private final EdqsClientQueueFactory queueFactory; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java index c4ce0c7a7e..dc8d4bf36d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @Service @RequiredArgsConstructor -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") public class EdqsListener { private final EdqsService edqsService; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index c2750ae840..80aeee8635 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.edqs; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -42,7 +43,6 @@ import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.tenant.TenantDao; import java.util.List; import java.util.Map; @@ -58,11 +58,13 @@ import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; @Slf4j public abstract class EdqsSyncService { + @Value("${queue.edqs.sync.entity_batch_size:10000}") + private int entityBatchSize; + @Value("${queue.edqs.sync.ts_batch_size:10000}") + private int tsBatchSize; @Autowired private EntityDaoRegistry entityDaoRegistry; @Autowired - private TenantDao tenantDao; - @Autowired private AttributesDao attributesDao; @Autowired private KeyDictionaryDao keyDictionaryDao; @@ -112,7 +114,7 @@ public abstract class EdqsSyncService { Dao dao = entityDaoRegistry.getDao(entityType); UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); while (true) { - var batch = dao.findNextBatch(lastId, 10000); + var batch = dao.findNextBatch(lastId, entityBatchSize); if (batch.isEmpty()) { break; } @@ -140,7 +142,7 @@ public abstract class EdqsSyncService { while (true) { List batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, - lastRelationType, lastToEntityId, lastToEntityType, 10000); + lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize); if (batch.isEmpty()) { break; } @@ -189,7 +191,7 @@ public abstract class EdqsSyncService { int lastAttributeKey = Integer.MIN_VALUE; while (true) { - List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, 10000); + List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize); if (batch.isEmpty()) { break; } @@ -228,7 +230,7 @@ public abstract class EdqsSyncService { int lastKey = Integer.MIN_VALUE; while (true) { - List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize); if (batch.isEmpty()) { break; } diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index f4e9a02b45..0c6f948770 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -27,7 +27,7 @@ import java.util.Collections; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") public class KafkaEdqsSyncService extends EdqsSyncService { private final TbKafkaSettings kafkaSettings; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java index 924bf94830..11d5894307 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java @@ -22,7 +22,7 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") public class LocalEdqsSyncService extends EdqsSyncService { private final EdqsRocksDb db; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index dd0e5948c7..e41d0c2aed 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1695,7 +1695,10 @@ queue: # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" edqs: - sync_enabled: "${TB_EDQS_SYNC_ENABLED:true}" # FIXME: disable by default before release + sync: + enabled: "${TB_EDQS_SYNC_ENABLED:true}" # Enable/disable EDQS synchronization with postgres db FIXME: disable by default before release + entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" # batch size of entities being synced with EDQS + ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" # batch size of timeseries data being synced with EDQS api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release mode: "${TB_EDQS_MODE:local}" # local or remote local: diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index baf9f98993..9bb0bbe30a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -36,7 +36,7 @@ import static org.awaitility.Awaitility.await; @TestPropertySource(properties = { // "queue.type=kafka", // uncomment to use Kafka // "queue.kafka.bootstrap.servers=10.7.1.254:9092", - "queue.edqs.sync_enabled=true", + "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" }) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index cf24deb7e4..cb377e0431 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -34,7 +34,7 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { - "queue.edqs.sync_enabled=true", + "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" }) diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index ba6863c705..8933c36db5 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -57,5 +57,5 @@ server.log_controller_error_stack_trace=true transport.gateway.dashboard.sync.enabled=false -queue.edqs.sync_enabled=false +queue.edqs.sync.enabled=false queue.edqs.api_enabled=false diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java index 838f9b4aa2..27a1e09f79 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -22,7 +22,7 @@ import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) // TODO: tb-core ? -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + "'${queue.edqs.mode:null}'=='local'))") public @interface EdqsComponent { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java index 5055787fde..3e04dda3b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") public @interface InMemoryEdqsComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java index 9f112e7f59..9963972e9c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java @@ -21,7 +21,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + "'${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='kafka'))") public @interface KafkaEdqsComponent { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java index 91934293a6..b69aa52ec5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.edqs.EdqsService; @Service -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "false", matchIfMissing = true) +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "false", matchIfMissing = true) public class DummyEdqsService implements EdqsService { @Override From 3d9897f5af21ae26f197009cd6e985e5316f10ee Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 13:30:42 +0200 Subject: [PATCH 4/6] fixed NullPointer exception in CachedAttributesService --- .../server/dao/attributes/CachedAttributesService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index c5ac50c9f1..3c12ae3209 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -266,7 +266,9 @@ public class CachedAttributesService implements AttributesService { String key = keyVersionPair.getFirst(); Long version = keyVersionPair.getSecond(); cache.evict(new AttributeCacheKey(scope, entityId, key), version); - edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } return key; }, cacheExecutor)).collect(Collectors.toList())); } From 09bd3de984d15c7596bdc7d5152b3c5bc91ec342 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 16:42:31 +0200 Subject: [PATCH 5/6] test fixes --- .../controller/EntityQueryControllerTest.java | 16 ++-------------- .../discovery/HashPartitionServiceTest.java | 1 + .../state/DefaultDeviceStateServiceTest.java | 18 +++++++++--------- .../update/330/device_profile_001_out.json | 6 ++++-- 4 files changed, 16 insertions(+), 25 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 508f66e584..d9652946ab 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -817,26 +817,14 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(1, data.getTotalElements()); - Assert.assertEquals(1, data.getTotalPages()); - Assert.assertEquals(1, data.getData().size()); + findByQueryAndCheck(query, 1); // unnassign dashboard login(TENANT_EMAIL, TENANT_PASSWORD); doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD); - PageData dataAfterUnassign = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(0, dataAfterUnassign.getTotalElements()); - Assert.assertEquals(0, dataAfterUnassign.getTotalPages()); - Assert.assertEquals(0, dataAfterUnassign.getData().size()); + findByQueryAndCheck(query, 0); } private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 0a4009e8b2..0d3ae093aa 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -431,6 +431,7 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge"); ReflectionTestUtils.setField(partitionService, "edgePartitions", 10); + ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12); partitionService.init(); partitionService.partitionsInit(); return partitionService; diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 9880ec964b..3c3fe2716c 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -211,7 +211,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastConnectTime) @@ -298,7 +298,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastDisconnectTime) @@ -421,13 +421,13 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(lastInactivityTime) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -465,12 +465,12 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -1002,7 +1002,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.isActive()).isFalse(); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime) @@ -1170,7 +1170,7 @@ public class DefaultDeviceStateServiceTest { assertThat(attributeRequestCaptor.getAllValues()).hasSize(2) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { @@ -1179,7 +1179,7 @@ public class DefaultDeviceStateServiceTest { }); }) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { diff --git a/application/src/test/resources/update/330/device_profile_001_out.json b/application/src/test/resources/update/330/device_profile_001_out.json index 9a349c6638..29e2241ee9 100644 --- a/application/src/test/resources/update/330/device_profile_001_out.json +++ b/application/src/test/resources/update/330/device_profile_001_out.json @@ -64,7 +64,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } @@ -103,7 +104,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } From f128f49d44edbdafbe6927b3137a3106cdc33bf7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 18:57:23 +0200 Subject: [PATCH 6/6] test fixes --- .../service/entitiy/EntityServiceTest.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index fab4cba3f5..a559eca6ce 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -1178,8 +1178,7 @@ public class EntityServiceTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); + PageData result = findByQueryAndCheck(query, 1); String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName).isEqualTo(devices.get(0).getName()); @@ -1240,11 +1239,8 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setRootEntity(asset.getId()); EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - PageData relationsResult = entityService.findEntityDataByQuery(tenantId, customer.getId(), query); - long relationsResultCnt = entityService.countEntitiesByQuery(tenantId, customer.getId(), query); - - Assert.assertEquals(relationsCnt, relationsResult.getData().size()); - Assert.assertEquals(relationsCnt, relationsResultCnt); + findByQueryAndCheck(customer.getId(), query, relationsCnt); + countByQueryAndCheck(customer.getId(), query, relationsCnt); } } @@ -1497,9 +1493,6 @@ public class EntityServiceTest extends AbstractControllerTest { assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); } - private PageData searchEntities(EntityDataQuery query) { - return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - } private EntityDataQuery createDeviceSearchQuery(String deviceField, StringOperation operation, String searchQuery) { DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); @@ -1593,7 +1586,7 @@ public class EntityServiceTest extends AbstractControllerTest { .getLatest().get(currentAttributeKeyType).get("temperature").getValue()); } List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + assertThat(loadedTemperatures).containsExactlyInAnyOrderElementsOf(deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45); @@ -1614,8 +1607,7 @@ public class EntityServiceTest extends AbstractControllerTest { entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); - + assertThat(loadedHighTemperatures).containsExactlyInAnyOrderElementsOf(deviceHighTemperatures); } deviceService.deleteDevicesByTenantId(tenantId); }