Browse Source

Merge pull request #12580 from dashevchenko/feature/edqs/testFixes

Keyset pagination for tenant entities
pull/12721/head
Viacheslav Klimov 2 years ago
committed by GitHub
parent
commit
342280105a
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 2
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java
  2. 2
      application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java
  3. 107
      application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java
  4. 2
      application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java
  5. 2
      application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java
  6. 5
      application/src/main/resources/thingsboard.yml
  7. 7
      application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java
  8. 40
      application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java
  9. 1
      application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java
  10. 13
      application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java
  11. 262
      application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java
  12. 18
      application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java
  13. 2
      application/src/test/resources/application-test.properties
  14. 6
      application/src/test/resources/update/330/device_profile_001_out.json
  15. 17
      common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java
  16. 32
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java
  17. 6
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java
  18. 7
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java
  19. 2
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java
  20. 4
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java
  21. 4
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java
  22. 3
      common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java
  23. 8
      common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java
  24. 4
      common/edqs/pom.xml
  25. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java
  26. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java
  27. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java
  28. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java
  29. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java
  30. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java
  31. 58
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java
  32. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java
  33. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java
  34. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java
  35. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java
  36. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java
  37. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java
  38. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java
  39. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java
  40. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java
  41. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java
  42. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java
  43. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java
  44. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java
  45. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java
  46. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java
  47. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java
  48. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java
  49. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java
  50. 9
      common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java
  51. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java
  52. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java
  53. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java
  54. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java
  55. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java
  56. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java
  57. 4
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java
  58. 75
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java
  59. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java
  60. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java
  61. 4
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java
  62. 71
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java
  63. 112
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java
  64. 53
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java
  65. 116
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java
  66. 35
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java
  67. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java
  68. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java
  69. 25
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java
  70. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java
  71. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java
  72. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java
  73. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java
  74. 121
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java
  75. 96
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java
  76. 84
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java
  77. 83
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java
  78. 34
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java
  79. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java
  80. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java
  81. 8
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java
  82. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java
  83. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java
  84. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java
  85. 29
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java
  86. 24
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java
  87. 33
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java
  88. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java
  89. 37
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java
  90. 36
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java
  91. 91
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java
  92. 7
      common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java
  93. 22
      common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java
  94. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java
  95. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java
  96. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java
  97. 173
      common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java
  98. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java
  99. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java
  100. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java

2
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java

@ -81,7 +81,7 @@ import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@Slf4j
@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true")
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true")
public class DefaultEdqsService implements EdqsService {
private final EdqsClientQueueFactory queueFactory;

2
application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java

@ -28,7 +28,7 @@ import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true")
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true")
public class EdqsListener {
private final EdqsService edqsService;

107
application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.edqs;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
@ -27,7 +28,6 @@ import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.LatestTsKv;
import org.thingsboard.server.common.data.edqs.fields.EntityFields;
import org.thingsboard.server.common.data.edqs.fields.TenantFields;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
@ -37,65 +37,40 @@ import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.attributes.AttributesDao;
import org.thingsboard.server.dao.dictionary.KeyDictionaryDao;
import org.thingsboard.server.dao.entity.EntityDaoRegistry;
import org.thingsboard.server.dao.group.EntityGroupDao;
import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
import org.thingsboard.server.dao.model.sql.RelationEntity;
import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry;
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import org.thingsboard.server.dao.sql.relation.RelationRepository;
import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository;
import org.thingsboard.server.dao.tenant.TenantDao;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.common.data.ObjectType.API_USAGE_STATE;
import static org.thingsboard.server.common.data.ObjectType.ASSET;
import static org.thingsboard.server.common.data.ObjectType.ASSET_PROFILE;
import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV;
import static org.thingsboard.server.common.data.ObjectType.BLOB_ENTITY;
import static org.thingsboard.server.common.data.ObjectType.CONVERTER;
import static org.thingsboard.server.common.data.ObjectType.CUSTOMER;
import static org.thingsboard.server.common.data.ObjectType.DASHBOARD;
import static org.thingsboard.server.common.data.ObjectType.DEVICE;
import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE;
import static org.thingsboard.server.common.data.ObjectType.EDGE;
import static org.thingsboard.server.common.data.ObjectType.ENTITY_GROUP;
import static org.thingsboard.server.common.data.ObjectType.ENTITY_VIEW;
import static org.thingsboard.server.common.data.ObjectType.INTEGRATION;
import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV;
import static org.thingsboard.server.common.data.ObjectType.QUEUE_STATS;
import static org.thingsboard.server.common.data.ObjectType.RELATION;
import static org.thingsboard.server.common.data.ObjectType.ROLE;
import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN;
import static org.thingsboard.server.common.data.ObjectType.SCHEDULER_EVENT;
import static org.thingsboard.server.common.data.ObjectType.TENANT;
import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE;
import static org.thingsboard.server.common.data.ObjectType.USER;
import static org.thingsboard.server.common.data.ObjectType.WIDGETS_BUNDLE;
import static org.thingsboard.server.common.data.ObjectType.WIDGET_TYPE;
import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes;
@Slf4j
public abstract class EdqsSyncService {
@Value("${queue.edqs.sync.entity_batch_size:10000}")
private int entityBatchSize;
@Value("${queue.edqs.sync.ts_batch_size:10000}")
private int tsBatchSize;
@Autowired
private EntityDaoRegistry entityDaoRegistry;
@Autowired
private TenantDao tenantDao;
@Autowired
private AttributesDao attributesDao;
@Autowired
private KeyDictionaryDao keyDictionaryDao;
@Autowired
private RelationRepository relationRepository;
@Autowired
private EntityGroupDao entityGroupDao;
@Autowired
private TsKvLatestRepository tsKvLatestRepository;
@Autowired
@Lazy
@ -106,12 +81,6 @@ public abstract class EdqsSyncService {
private final Map<ObjectType, AtomicInteger> counters = new ConcurrentHashMap<>();
public static final Set<ObjectType> edqsTenantTypes = EnumSet.of(
TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD,
RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE,
BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS
);
public abstract boolean isSyncNeeded();
public void sync() {
@ -119,9 +88,7 @@ public abstract class EdqsSyncService {
long startTs = System.currentTimeMillis();
counters.clear();
syncTenants();
syncTenantEntities();
syncEntityGroups();
syncRelations();
loadKeyDictionary();
syncAttributes();
@ -139,49 +106,28 @@ public abstract class EdqsSyncService {
edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object);
}
private void syncTenants() {
log.info("Synchronizing tenants to EDQS");
long ts = System.currentTimeMillis();
var tenants = new PageDataIterable<>(tenantDao::findAllFields, 10000);
for (EntityFields entityFields : tenants) {
TenantId tenantId = TenantId.fromUUID(entityFields.getId());
entityInfoMap.put(entityFields.getId(), new EntityIdInfo(EntityType.TENANT, tenantId));
process(tenantId, TENANT, new Entity(EntityType.TENANT, entityFields));
}
process(TenantId.SYS_TENANT_ID, TENANT, new Entity(EntityType.TENANT, new TenantFields(TenantId.SYS_TENANT_ID.getId(), Long.MAX_VALUE)));
log.info("Finished synchronizing tenants to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void syncTenantEntities() {
for (ObjectType type : edqsTenantTypes) {
log.info("Synchronizing tenant {} entities to EDQS", type);
log.info("Synchronizing {} entities to EDQS", type);
long ts = System.currentTimeMillis();
EntityType entityType = type.toEntityType();
Dao<?> dao = entityDaoRegistry.getDao(entityType);
var entities = new PageDataIterable<>(dao::findAllFields, 10000);
for (EntityFields entityFields : entities) {
TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId());
entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId));
process(tenantId, type, new Entity(type.toEntityType(), entityFields));
}
log.info("Finished synchronizing tenant {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts));
}
}
private void syncEntityGroups() {
log.info("Synchronizing entity groups to EDQS");
long ts = System.currentTimeMillis();
var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 30000);
for (EntityFields groupFields : entityGroups) {
EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId());
if (entityIdInfo != null) {
entityInfoMap.put(groupFields.getId(), new EntityIdInfo(EntityType.ENTITY_GROUP, entityIdInfo.tenantId()));
process(entityIdInfo.tenantId(), ENTITY_GROUP, new Entity(EntityType.ENTITY_GROUP, groupFields));
} else {
log.info("Entity group owner not found: " + groupFields.getOwnerId());
UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000");
while (true) {
var batch = dao.findNextBatch(lastId, entityBatchSize);
if (batch.isEmpty()) {
break;
}
for (EntityFields entityFields : batch) {
TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId());
entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId));
process(tenantId, type, new Entity(entityType, entityFields));
}
EntityFields lastRecord = batch.get(batch.size() - 1);
lastId = lastRecord.getId();
}
log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts));
}
log.info("Finished synchronizing entity groups to EDQS in {} ms", (System.currentTimeMillis() - ts));
}
private void syncRelations() {
@ -196,7 +142,7 @@ public abstract class EdqsSyncService {
while (true) {
List<RelationEntity> batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup,
lastRelationType, lastToEntityId, lastToEntityType, 10000);
lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize);
if (batch.isEmpty()) {
break;
}
@ -215,7 +161,7 @@ public abstract class EdqsSyncService {
private void processRelationBatch(List<RelationEntity> relations) {
for (RelationEntity relation : relations) {
if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup()) || (RelationTypeGroup.FROM_ENTITY_GROUP.name().equals(relation.getRelationTypeGroup()))) {
if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) {
EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId());
if (entityIdInfo != null) {
process(entityIdInfo.tenantId(), RELATION, relation.toData());
@ -245,7 +191,7 @@ public abstract class EdqsSyncService {
int lastAttributeKey = Integer.MIN_VALUE;
while (true) {
List<AttributeKvEntity> batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, 10000);
List<AttributeKvEntity> batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize);
if (batch.isEmpty()) {
break;
}
@ -284,7 +230,7 @@ public abstract class EdqsSyncService {
int lastKey = Integer.MIN_VALUE;
while (true) {
List<TsKvLatestEntity> batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000);
List<TsKvLatestEntity> batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize);
if (batch.isEmpty()) {
break;
}
@ -332,6 +278,7 @@ public abstract class EdqsSyncService {
return strKey;
}
public record EntityIdInfo(EntityType entityType, TenantId tenantId) {}
public record EntityIdInfo(EntityType entityType, TenantId tenantId) {
}
}

2
application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java

@ -27,7 +27,7 @@ import java.util.Collections;
@Service
@RequiredArgsConstructor
@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'")
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'")
public class KafkaEdqsSyncService extends EdqsSyncService {
private final TbKafkaSettings kafkaSettings;

2
application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java

@ -22,7 +22,7 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb;
@Service
@RequiredArgsConstructor
@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'")
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'")
public class LocalEdqsSyncService extends EdqsSyncService {
private final EdqsRocksDb db;

5
application/src/main/resources/thingsboard.yml

@ -1695,7 +1695,10 @@ queue:
# Statistics printing interval for Housekeeper
print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}"
edqs:
sync_enabled: "${TB_EDQS_SYNC_ENABLED:true}" # FIXME: disable by default before release
sync:
enabled: "${TB_EDQS_SYNC_ENABLED:true}" # Enable/disable EDQS synchronization with postgres db FIXME: disable by default before release
entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" # batch size of entities being synced with EDQS
ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" # batch size of timeseries data being synced with EDQS
api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release
mode: "${TB_EDQS_MODE:local}" # local or remote
local:

7
application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java

@ -28,14 +28,15 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import static org.awaitility.Awaitility.await;
@DaoSqlTest
@TestPropertySource(properties = {
"queue.type=kafka", // uncomment to use Kafka
"queue.kafka.bootstrap.servers=10.7.1.254:9092",
"queue.edqs.sync_enabled=true",
// "queue.type=kafka", // uncomment to use Kafka
// "queue.kafka.bootstrap.servers=10.7.1.254:9092",
"queue.edqs.sync.enabled=true",
"queue.edqs.api_enabled=true",
"queue.edqs.mode=local"
})

40
application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@ -73,6 +73,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@ -162,12 +163,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityTypeFilter allDeviceFilter = new EntityTypeFilter();
allDeviceFilter.setEntityType(EntityType.DEVICE);
EntityCountQuery query = new EntityCountQuery(allDeviceFilter);
Long initialCount = countByQuery(query);
countByQueryAndCheck(query, 0);
loginTenantAdmin();
List<Device> devices = new ArrayList<>();
String devicePrefix = "Device" + RandomStringUtils.random(5);
String devicePrefix = "Device" + RandomStringUtils.randomAlphabetic(5);
for (int i = 0; i < 97; i++) {
Device device = new Device();
device.setName(devicePrefix + i);
@ -177,18 +178,18 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
Thread.sleep(1);
}
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("");
loginSysAdmin();
EntityCountQuery countQuery = new EntityCountQuery(filter);
countByQueryAndCheck(countQuery, initialCount + 97);
countByQueryAndCheck(countQuery, 97);
filter.setDeviceType("unknown");
filter.setDeviceTypes(List.of("unknown"));
countByQueryAndCheck(countQuery, 0);
filter.setDeviceType("default");
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter(devicePrefix + "1");
countByQueryAndCheck(countQuery, 11);
@ -199,7 +200,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
countQuery = new EntityCountQuery(entityListFilter);
countByQueryAndCheck(countQuery, 97);
countByQueryAndCheck(query, initialCount + 97);
countByQueryAndCheck(countQuery, 97);
}
@Test
@ -816,26 +817,14 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
PageData<EntityData> data =
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(1, data.getTotalElements());
Assert.assertEquals(1, data.getTotalPages());
Assert.assertEquals(1, data.getData().size());
findByQueryAndCheck(query, 1);
// unnassign dashboard
login(TENANT_EMAIL, TENANT_PASSWORD);
doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD);
PageData<EntityData> dataAfterUnassign =
doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
});
Assert.assertEquals(0, dataAfterUnassign.getTotalElements());
Assert.assertEquals(0, dataAfterUnassign.getTotalPages());
Assert.assertEquals(0, dataAfterUnassign.getData().size());
findByQueryAndCheck(query, 0);
}
private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception {
@ -847,9 +836,9 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
var loadedEntities = new ArrayList<>(data.getData());
return loadedEntities.size() == expectedNumOfDevices;
});
if (expectedNumOfDevices == 0) {
return;
}
if (expectedNumOfDevices == 0) {
return;
}
var data = findByQuery(query);
var loadedEntities = new ArrayList<>(data.getData());
@ -870,7 +859,8 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
}
protected PageData<EntityData> findByQuery(EntityDataQuery query) throws Exception {
return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {
});
}
protected PageData<EntityData> findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception {

1
application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java

@ -431,6 +431,7 @@ public class HashPartitionServiceTest {
ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName);
ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge");
ReflectionTestUtils.setField(partitionService, "edgePartitions", 10);
ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12);
partitionService.init();
partitionService.partitionsInit();
return partitionService;

13
application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@ -21,7 +21,6 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.permission.MergedUserPermissions;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
@ -35,7 +34,7 @@ import static org.awaitility.Awaitility.await;
@DaoSqlTest
@TestPropertySource(properties = {
"queue.edqs.sync_enabled=true",
"queue.edqs.sync.enabled=true",
"queue.edqs.api_enabled=true",
"queue.edqs.mode=local"
})
@ -53,19 +52,19 @@ public class EdqsEntityServiceTest extends EntityServiceTest {
}
@Override
protected PageData<EntityData> findByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityDataQuery query, long expectedResultSize) {
return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, permissions, query),
protected PageData<EntityData> findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) {
return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query),
result -> result.getTotalElements() == expectedResultSize);
}
@Override
protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) {
return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), mergedUserPermissionsPE, countQuery, expectedResult);
return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult);
}
@Override
protected long countByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityCountQuery query, int expectedResult) {
return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, permissions, query),
protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) {
return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query),
result -> result == expectedResult);
}

262
application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java

@ -489,7 +489,7 @@ public class EntityServiceTest extends AbstractControllerTest {
filter.setFilters(Lists.newArrayList(
new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)),
new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE))));
countByQueryAndCheck(countQuery, 9);
countByQueryAndCheck(countQuery, 3);
deviceService.deleteDevicesByTenantId(tenantId);
assetService.deleteAssetsByTenantId(tenantId);
@ -1178,8 +1178,7 @@ public class EntityServiceTest extends AbstractControllerTest {
EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null);
PageData<EntityData> result = searchEntities(query);
assertEquals(1, result.getTotalElements());
PageData<EntityData> result = findByQueryAndCheck(query, 1);
String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue();
assertThat(deviceName).isEqualTo(devices.get(0).getName());
@ -1240,11 +1239,8 @@ public class EntityServiceTest extends AbstractControllerTest {
filter.setRootEntity(asset.getId());
EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString);
PageData<EntityData> relationsResult = entityService.findEntityDataByQuery(tenantId, customer.getId(), query);
long relationsResultCnt = entityService.countEntitiesByQuery(tenantId, customer.getId(), query);
Assert.assertEquals(relationsCnt, relationsResult.getData().size());
Assert.assertEquals(relationsCnt, relationsResultCnt);
findByQueryAndCheck(customer.getId(), query, relationsCnt);
countByQueryAndCheck(customer.getId(), query, relationsCnt);
}
}
@ -1444,30 +1440,16 @@ public class EntityServiceTest extends AbstractControllerTest {
assertThat(deviceName).isEqualTo(customerDevices.get(0).getName());
// find by customer user with generic permission
MergedUserPermissions mergedGenericPermission = new MergedUserPermissions(Map.of(Resource.DEVICE, Set.of(Operation.READ)), Collections.emptyMap());
PageData<EntityData> customerResults = findByQueryAndCheck(customerId, mergedGenericPermission, query, 1);
PageData<EntityData> customerResults = findByQueryAndCheck(customerId, query, 1);
String cutomerDeviceName = customerResults.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue();
assertThat(cutomerDeviceName).isEqualTo(customerDevices.get(0).getName());
// find by customer user with group permission
MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(customerDeviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ))));
PageData<EntityData> result2 = findByQueryAndCheck(customerId, mergedGroupOnlyPermission, query, 1);
String resultDeviceName2 = result2.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue();
assertThat(resultDeviceName2).isEqualTo(customerDevices.get(0).getName());
// try to find tenant device by customer user
SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter();
tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId());
EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null);
findByQueryAndCheck(customerId, mergedGenericPermission, customerQuery2, 0);
// find by tenant user with group permission
PageData<EntityData> results3 = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), mergedGroupOnlyPermission, query, 1);
String deviceName3 = results3.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue();
assertThat(deviceName3).isEqualTo(customerDevices.get(0).getName());
findByQueryAndCheck(customerId, customerQuery2, 0);
}
private List<DeviceId> getResultDeviceIds(PageData<EntityData> result) {
@ -1511,9 +1493,6 @@ public class EntityServiceTest extends AbstractControllerTest {
assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME);
}
private PageData<EntityData> searchEntities(EntityDataQuery query) {
return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
}
private EntityDataQuery createDeviceSearchQuery(String deviceField, StringOperation operation, String searchQuery) {
DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter();
@ -1607,7 +1586,7 @@ public class EntityServiceTest extends AbstractControllerTest {
.getLatest().get(currentAttributeKeyType).get("temperature").getValue());
}
List<String> deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
assertThat(loadedTemperatures).containsExactlyInAnyOrderElementsOf(deviceTemperatures);
pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45);
@ -1628,236 +1607,11 @@ public class EntityServiceTest extends AbstractControllerTest {
entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList());
Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
assertThat(loadedHighTemperatures).containsExactlyInAnyOrderElementsOf(deviceHighTemperatures);
}
deviceService.deleteDevicesByTenantId(tenantId);
}
@Test
public void testFindEntityDataByRelationQuery_blobEntity_customerLevel() {
final int deviceCnt = 2;
final int relationsCnt = 3;
final int blobEntitiesCnt = deviceCnt * relationsCnt;
Customer customer = new Customer();
customer.setTenantId(tenantId);
customer.setTitle("Customer Relation Query");
customer = customerService.saveCustomer(customer);
List<Device> devices = new ArrayList<>();
for (int i = 0; i < deviceCnt; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device relation query " + i);
device.setCustomerId(customer.getId());
device.setType("default");
devices.add(deviceService.saveDevice(device));
}
List<BlobEntity> blobEntities = new ArrayList<>();
for (int i = 0; i < blobEntitiesCnt; i++) {
BlobEntity blobEntity = new BlobEntity();
blobEntity.setName("Blob relation query " + i);
blobEntity.setTenantId(tenantId);
blobEntity.setContentType("image/png");
blobEntity.setData(ByteBuffer.allocate(1024));
blobEntity.setCustomerId(customer.getId());
blobEntity.setType("Report");
blobEntities.add(blobEntityService.saveBlobEntity(blobEntity));
}
for (int i = 0; i < deviceCnt; i++) {
for (int j = 0; j < relationsCnt; j++) {
EntityRelation relationEntity = new EntityRelation();
relationEntity.setFrom(devices.get(i).getId());
relationEntity.setTo(blobEntities.get(j + (i * relationsCnt)).getId());
relationEntity.setTypeGroup(RelationTypeGroup.COMMON);
relationEntity.setType("fileAttached");
relationService.saveRelation(tenantId, relationEntity);
}
}
MergedUserPermissions mergedUserPermissions = new MergedUserPermissions(Map.of(ALL, Set.of(Operation.ALL)), Collections.emptyMap());
RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("fileAttached", Collections.singletonList(EntityType.BLOB_ENTITY));
RelationsQueryFilter filter = new RelationsQueryFilter();
filter.setFilters(Collections.singletonList(relationEntityTypeFilter));
filter.setDirection(EntitySearchDirection.FROM);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null);
for (Device device : devices) {
filter.setRootEntity(device.getId());
EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
findByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt);
countByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt);
/*
In order to be careful with updating Relation Query while adding new Entity Type,
this checkup will help to find place, where you could check the correctness of building query
*/
Assert.assertEquals(38, EntityType.values().length);
}
}
@Test
public void testFindEntitiesByRelationEntityTypeFilterWithTenantGroupPermission() {
final int assetCount = 2;
final int relationsCnt = 4;
final int deviceEntitiesCnt = assetCount * relationsCnt;
EntityGroup deviceGroup = new EntityGroup();
deviceGroup.setName("Device Tenant Level Group");
deviceGroup.setOwnerId(tenantId);
deviceGroup.setTenantId(tenantId);
deviceGroup.setType(EntityType.DEVICE);
deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup);
List<Asset> assets = new ArrayList<>();
for (int i = 0; i < assetCount; i++) {
Asset building = new Asset();
building.setTenantId(tenantId);
building.setName("Building _" + i);
building.setType("building");
building = assetService.saveAsset(building);
assets.add(building);
}
List<Device> devices = new ArrayList<>();
for (int i = 0; i < deviceEntitiesCnt; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Test device " + i);
device.setType("default");
Device savedDevice = deviceService.saveDevice(device);
devices.add(savedDevice);
if (i % 2 == 0) {
entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId());
}
}
for (int i = 0; i < assetCount; i++) {
for (int j = 0; j < relationsCnt; j++) {
EntityRelation relationEntity = new EntityRelation();
relationEntity.setFrom(assets.get(i).getId());
relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId());
relationEntity.setTypeGroup(RelationTypeGroup.COMMON);
relationEntity.setType("contains");
relationService.saveRelation(tenantId, relationEntity);
}
}
MergedUserPermissions groupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(),
Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ))));
RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE));
RelationsQueryFilter filter = new RelationsQueryFilter();
filter.setFilters(Collections.singletonList(relationEntityTypeFilter));
filter.setDirection(EntitySearchDirection.FROM);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null);
List<KeyFilter> keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device ");
for (Asset asset : assets) {
filter.setRootEntity(asset.getId());
EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString);
findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2);
countByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2);
}
}
@Test
public void testFindEntitiesWithRelationEntityTypeFilterByCustomerUser() {
Customer customer = new Customer();
customer.setTenantId(tenantId);
customer.setTitle("Customer Relation Query");
customer = customerService.saveCustomer(customer);
final int assetCount = 2;
final int relationsCnt = 4;
final int deviceEntitiesCnt = assetCount * relationsCnt;
EntityGroup deviceGroup = new EntityGroup();
deviceGroup.setName("Device Tenant Level Group");
deviceGroup.setOwnerId(customer.getId());
deviceGroup.setTenantId(tenantId);
deviceGroup.setType(EntityType.DEVICE);
deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup);
List<Asset> assets = new ArrayList<>();
for (int i = 0; i < assetCount; i++) {
Asset building = new Asset();
building.setTenantId(tenantId);
building.setCustomerId(customer.getId());
building.setName("Building _" + i);
building.setType("building");
building = assetService.saveAsset(building);
assets.add(building);
}
List<Device> devices = new ArrayList<>();
for (int i = 0; i < deviceEntitiesCnt; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setCustomerId(customer.getId());
device.setName("Test device " + i);
device.setType("default");
Device savedDevice = deviceService.saveDevice(device);
devices.add(savedDevice);
if (i % 2 == 0) {
entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId());
}
}
for (int i = 0; i < assetCount; i++) {
for (int j = 0; j < relationsCnt; j++) {
EntityRelation relationEntity = new EntityRelation();
relationEntity.setFrom(assets.get(i).getId());
relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId());
relationEntity.setTypeGroup(RelationTypeGroup.COMMON);
relationEntity.setType("contains");
relationService.saveRelation(tenantId, relationEntity);
}
}
MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL))));
MergedUserPermissions mergedGenericOnlyPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.emptyMap());
MergedUserPermissions mergedGenericAndGroupPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL))));
RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE));
RelationsQueryFilter filter = new RelationsQueryFilter();
filter.setFilters(Collections.singletonList(relationEntityTypeFilter));
filter.setDirection(EntitySearchDirection.FROM);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null);
List<KeyFilter> keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device ");
EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString);
for (Asset asset : assets) {
filter.setRootEntity(asset.getId());
//check by user with generic permission
PageData<EntityData> relationsResult = findByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt);
countByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt);
//check by user with generic and group permission
PageData<EntityData> relationsResult1 = findByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt);
countByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt);
//check by other customer user with group only permission
PageData<EntityData> relationsResult2 = findByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2);
long relationsResultCnt2 = countByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2);
Assert.assertEquals(relationsCnt / 2, relationsResult2.getData().size());
Assert.assertEquals(relationsCnt / 2, relationsResultCnt2);
//check by other customer user with generic and group only permission
PageData<EntityData> relationsResult3 = findByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2);
long relationsResultCnt3 = countByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2);
Assert.assertEquals(relationsCnt / 2, relationsResult3.getData().size());
Assert.assertEquals(relationsCnt / 2, relationsResultCnt3);
}
}
@Test
public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException {

18
application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java

@ -211,7 +211,7 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) &&
request.getEntries().get(0).getValue().equals(lastConnectTime)
@ -298,7 +298,7 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) &&
request.getEntries().get(0).getValue().equals(lastDisconnectTime)
@ -421,13 +421,13 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) &&
request.getEntries().get(0).getValue().equals(lastInactivityTime)
));
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) &&
request.getEntries().get(0).getValue().equals(false)
@ -465,12 +465,12 @@ public class DefaultDeviceStateServiceTest {
// THEN
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME)
));
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) &&
request.getEntries().get(0).getValue().equals(false)
@ -1002,7 +1002,7 @@ public class DefaultDeviceStateServiceTest {
assertThat(actualNotification.isActive()).isFalse();
then(telemetrySubscriptionService).should().saveAttributes(argThat(request ->
request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) &&
request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) &&
request.getScope().equals(AttributeScope.SERVER_SCOPE) &&
request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) &&
request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime)
@ -1170,7 +1170,7 @@ public class DefaultDeviceStateServiceTest {
assertThat(attributeRequestCaptor.getAllValues()).hasSize(2)
.anySatisfy(request -> {
assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID);
assertThat(request.getTenantId()).isEqualTo(tenantId);
assertThat(request.getEntityId()).isEqualTo(deviceId);
assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> {
@ -1179,7 +1179,7 @@ public class DefaultDeviceStateServiceTest {
});
})
.anySatisfy(request -> {
assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID);
assertThat(request.getTenantId()).isEqualTo(tenantId);
assertThat(request.getEntityId()).isEqualTo(deviceId);
assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE);
assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> {

2
application/src/test/resources/application-test.properties

@ -57,5 +57,5 @@ server.log_controller_error_stack_trace=true
transport.gateway.dashboard.sync.enabled=false
queue.edqs.sync_enabled=false
queue.edqs.sync.enabled=false
queue.edqs.api_enabled=false

6
application/src/test/resources/update/330/device_profile_001_out.json

@ -64,7 +64,8 @@
"dynamicValue": {
"sourceType": null,
"sourceAttribute": null,
"inherit": false
"inherit": false,
"resolvedValue" : null
}
}
}
@ -103,7 +104,8 @@
"dynamicValue": {
"sourceType": null,
"sourceAttribute": null,
"inherit": false
"inherit": false,
"resolvedValue" : null
}
}
}

17
common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java

@ -30,16 +30,8 @@ public enum ObjectType {
RULE_CHAIN,
OTA_PACKAGE,
RESOURCE,
ROLE,
ENTITY_GROUP,
DEVICE_GROUP_OTA_PACKAGE,
GROUP_PERMISSION,
BLOB_ENTITY,
SCHEDULER_EVENT,
EVENT,
RULE_NODE,
CONVERTER,
INTEGRATION,
USER,
USER_CREDENTIALS,
USER_AUTH_SETTINGS,
@ -62,8 +54,6 @@ public enum ObjectType {
NOTIFICATION_TARGET,
NOTIFICATION_TEMPLATE,
NOTIFICATION_RULE,
WHITE_LABELING,
CUSTOM_TRANSLATION,
ALARM_COMMENT,
ALARM_TYPE,
API_USAGE_STATE,
@ -75,16 +65,15 @@ public enum ObjectType {
LATEST_TS_KV;
public static final Set<ObjectType> edqsTenantTypes = EnumSet.of(
TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD,
RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE,
BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS
TENANT, TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD,
RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS
);
public static final Set<ObjectType> edqsTypes = new HashSet<>(edqsTenantTypes);
public static final Set<ObjectType> edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD,
API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV);
static {
edqsTypes.addAll(Arrays.asList(TENANT, ENTITY_GROUP, RELATION, ATTRIBUTE_KV, LATEST_TS_KV));
edqsTypes.addAll(Arrays.asList(RELATION, ATTRIBUTE_KV, LATEST_TS_KV));
}
public EntityType toEntityType() {

32
common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java

@ -15,10 +15,15 @@
*/
package org.thingsboard.server.common.data.edqs.fields;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@ -27,7 +32,30 @@ import java.util.UUID;
@SuperBuilder
public class DashboardFields extends AbstractEntityFields {
public DashboardFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) {
super(id, createdTime, tenantId, customerId, name, version);
private static ObjectMapper objectMapper = new ObjectMapper();
private List<UUID> assignedCustomerIds;
public DashboardFields(UUID id, long createdTime, UUID tenantId, String assignedCustomers, String name, Long version) {
super(id, createdTime, tenantId, name, version);
this.assignedCustomerIds = getCustomerIds(assignedCustomers);
}
private static List<UUID> getCustomerIds(String assignedCustomers) {
List<UUID> ids = new ArrayList<>();
if (assignedCustomers == null || assignedCustomers.isEmpty()) {
return ids;
}
try {
JsonNode rootNode = objectMapper.readTree(assignedCustomers);
for (JsonNode node : rootNode) {
String idStr = node.path("customerId").path("id").asText();
if (!idStr.isEmpty()) {
ids.add(UUID.fromString(idStr));
}
}
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
return ids;
}
}

6
common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java

@ -19,6 +19,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
public interface EntityFields {
@ -37,6 +39,10 @@ public interface EntityFields {
return null;
}
default List<UUID> getAssignedCustomerIds() {
return Collections.emptyList();
}
default long getCreatedTime() {
return 0;
}

7
common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java

@ -19,6 +19,8 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.util.UUID;
@Data
@NoArgsConstructor
@SuperBuilder
@ -27,4 +29,9 @@ public class EntityViewFields extends AbstractEntityFields {
private String type;
private String additionalInfo;
public EntityViewFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, String type, String additionalInfo, Long version) {
super(id, createdTime, tenantId, customerId, name, version);
this.type = type;
this.additionalInfo = additionalInfo;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java

@ -85,7 +85,6 @@ public class FieldsUtil {
return CustomerFields.builder()
.id(entity.getUuidId())
.createdTime(entity.getCreatedTime())
.customerId(getCustomerId(entity.getCustomerId()))
.name(entity.getTitle())
.additionalInfo(getText(entity.getAdditionalInfo()))
.email(entity.getEmail())
@ -199,7 +198,6 @@ public class FieldsUtil {
return DashboardFields.builder()
.id(entity.getUuidId())
.createdTime(entity.getCreatedTime())
.customerId(getCustomerId(entity.getCustomerId()))
.name(entity.getTitle())
.version(entity.getVersion())
.build();

4
common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java

@ -15,12 +15,10 @@
*/
package org.thingsboard.server.common.data.edqs.query;
import com.fasterxml.jackson.annotation.JsonIncludeProperties;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.permission.MergedUserPermissions;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityDataQuery;
@ -32,7 +30,5 @@ public class EdqsRequest {
private EntityDataQuery entityDataQuery;
private EntityCountQuery entityCountQuery;
@JsonIncludeProperties({"genericPermissions", "groupPermissions"})
private MergedUserPermissions userPermissions;
}

4
common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java

@ -30,12 +30,10 @@ import java.util.Map;
public class QueryResult {
private final EntityId entityId;
private final boolean readAttrs;
private final boolean readTs;
private final Map<EntityKeyType, Map<String, TsValue>> latest;
public EntityData toOldEntityData() {
return new EntityData(entityId, readAttrs, readTs, latest, Collections.emptyMap(), Collections.emptyMap());
return new EntityData(entityId, latest, Collections.emptyMap(), Collections.emptyMap());
}
}

3
common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java

@ -44,4 +44,7 @@ public class QueryContext {
this(tenantId, customerId, entityType, false);
}
public boolean isTenantUser() {
return customerId == null || customerId.isNullUid();
}
}

8
common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java

@ -18,15 +18,13 @@ package org.thingsboard.server.common.data.queue;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasEntityType;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.id.QueueStatsId;
import org.thingsboard.server.common.data.id.TenantId;
@EqualsAndHashCode(callSuper = true)
@Data
public class QueueStats extends BaseData<QueueStatsId> implements HasTenantId, HasEntityType {
public class QueueStats extends BaseData<QueueStatsId> implements HasTenantId {
private TenantId tenantId;
private String queueName;
private String serviceId;
@ -38,8 +36,4 @@ public class QueueStats extends BaseData<QueueStatsId> implements HasTenantId, H
super(id);
}
@Override
public EntityType getEntityType() {
return EntityType.QUEUE_STATS;
}
}

4
common/edqs/pom.xml

@ -1,6 +1,6 @@
<!--
Copyright © 2016-2024 ThingsBoard, Inc.
Copyright © 2016-2024 The Thingsboard Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>4.0.0PE-SNAPSHOT</version>
<version>4.0.0-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

58
common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java

@ -1,58 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.data;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields;
import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class EntityGroupData extends BaseEntityData<EntityGroupFields> {
private final ConcurrentMap<UUID, EntityData<?>> entitiesById = new ConcurrentHashMap<>();
public EntityGroupData(UUID entityId) {
super(entityId);
}
@Override
public EntityType getEntityType() {
return EntityType.ENTITY_GROUP;
}
public Collection<EntityData<?>> getEntities() {
return entitiesById.values();
}
public boolean addOrUpdate(EntityData<?> ed) {
return entitiesById.put(ed.getId(), ed) == null;
}
public boolean remove(EntityData<?> ed) {
return entitiesById.remove(ed.getId()) != null;
}
public EntityData<?> getEntity(UUID entityId) {
return entitiesById.get(entityId);
}
public boolean remove(UUID toId) {
return entitiesById.remove(toId) != null;
}
}

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

9
common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -85,7 +85,8 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
private final EdqRepository repository;
private final EdqsConfig config;
private final EdqsPartitionService partitionService;
@Autowired @Lazy
@Autowired
@Lazy
private EdqsStateService stateService;
private MainQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>, QueueConfig> eventsConsumer;
@ -186,10 +187,10 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
if (request.getEntityDataQuery() != null) {
PageData<QueryResult> result = repository.findEntityDataByQuery(tenantId, customerId,
request.getUserPermissions(), request.getEntityDataQuery(), false);
request.getEntityDataQuery(), false);
response.setEntityDataQueryResult(result.mapData(QueryResult::toOldEntityData));
} else if (request.getEntityCountQuery() != null) {
long result = repository.countEntitiesByQuery(tenantId, customerId, request.getUserPermissions(), request.getEntityCountQuery(), tenantId.isSysTenantId());
long result = repository.countEntitiesByQuery(tenantId, customerId, request.getEntityCountQuery(), tenantId.isSysTenantId());
response.setEntityCountQueryResult(result);
}

2
common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

4
common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -27,8 +27,6 @@ public class SortableEntityData {
private final EntityData entityData;
private String sortValue;
private boolean readAttrs;
private boolean readTs;
public UUID getId(){
return entityData.getId();

75
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java

@ -1,75 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.edqs.data.EntityGroupData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
public abstract class AbstractEntityGroupQueryProcessor<T extends EntityFilter> extends AbstractSingleEntityTypeQueryProcessor<T> {
public AbstractEntityGroupQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) {
super(repo, ctx, query, filter);
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
var genericReadResults = processCustomerGenericRead(customerId, readAttrPermissions, readTsPermissions);
Map<UUID, SortableEntityData> mergedResult = new HashMap<>(genericReadResults.size());
for (SortableEntityData sd : genericReadResults) {
mergedResult.put(sd.getId(), sd);
}
for (GroupPermissions permissions : groupPermissions) {
SortableEntityData alreadyAdded = mergedResult.get(permissions.groupId);
if (alreadyAdded != null) {
alreadyAdded.setReadAttrs(alreadyAdded.isReadAttrs() || permissions.readAttrs);
alreadyAdded.setReadTs(alreadyAdded.isReadTs() || permissions.readTs);
} else {
EntityGroupData egData = repository.getEntityGroup(permissions.groupId);
if (matches(egData)) {
SortableEntityData sortData = toSortData(egData, permissions);
mergedResult.put(egData.getId(), sortData);
}
}
}
return new ArrayList<>(mergedResult.values());
}
@Override
protected CombinedPermissions getCombinedPermissionsInternal(UUID id, boolean read, boolean readAttrs, boolean readTs, List<GroupPermissions> groupPermissions) {
for (GroupPermissions eg : groupPermissions) {
if (read && readAttrs && readTs) {
break;
}
if (eg.groupId.equals(id)) {
read = true;
readAttrs = readAttrs || eg.readAttrs;
readTs = readTs || eg.readTs;
}
}
return new CombinedPermissions(read, readAttrs, readTs);
}
}

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

4
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -36,7 +36,7 @@ public abstract class AbstractEntitySearchQueryProcessor<T extends EntitySearchQ
@Override
public Set<UUID> getRootEntities() {
return Set.of(filter.getRootEntity().getId());
return Set.of(filter.getRootEntity().getId());
}
@Override

71
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,8 +15,7 @@
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.id.EntityGroupId;
import org.thingsboard.server.common.data.permission.MergedGroupTypePermissionInfo;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.edqs.data.EntityData;
@ -26,10 +25,7 @@ import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
@ -52,68 +48,12 @@ public abstract class AbstractQueryProcessor<T extends EntityFilter> implements
this.filter = filter;
}
protected CombinedPermissions getCombinedPermissions(UUID id, boolean genericRead, boolean genericAttrs, boolean genericTs, List<GroupPermissions> groupPermissions) {
return getCombinedPermissionsInternal(id, genericRead, genericRead && genericAttrs, genericRead && genericTs, groupPermissions);
}
protected CombinedPermissions getCombinedPermissions(UUID id, List<GroupPermissions> groupPermissions) {
return getCombinedPermissionsInternal(id, false, false, false, groupPermissions);
}
protected CombinedPermissions getCombinedPermissionsInternal(UUID id, boolean read, boolean readAttrs, boolean readTs, List<GroupPermissions> groupPermissions) {
for (GroupPermissions eg : groupPermissions) {
if (read && readAttrs && readTs) {
break;
}
boolean hasMorePermissions = !read || (!readAttrs && eg.readAttrs) || (!readTs && eg.readTs);
if (hasMorePermissions && repository.contains(eg.groupId, id)) {
read = true;
readAttrs = readAttrs || eg.readAttrs;
readTs = readTs || eg.readTs;
}
}
return new CombinedPermissions(read, readAttrs, readTs);
}
protected SortableEntityData toSortDataGroupsOnly(EntityData<?> ed, List<GroupPermissions> groupPermissions) {
SortableEntityData sortData;
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), groupPermissions);
if (permissions.isRead()) {
sortData = toSortData(ed, permissions);
} else {
sortData = null;
}
return sortData;
}
protected SortableEntityData toSortData(EntityData<?> ed, boolean readAttrs, boolean readTs) {
protected SortableEntityData toSortData(EntityData<?> ed) {
SortableEntityData sortData = new SortableEntityData(ed);
sortData.setSortValue(getSortValue(ed, sortKey));
sortData.setReadAttrs(readAttrs);
sortData.setReadTs(readTs);
return sortData;
}
protected SortableEntityData toSortData(EntityData<?> ed, Permissions permissions) {
return toSortData(ed, permissions.isReadAttrs(), permissions.isReadTs());
}
protected static List<GroupPermissions> toGroupPermissions(MergedGroupTypePermissionInfo readPermissions,
MergedGroupTypePermissionInfo readAttrPermissions,
MergedGroupTypePermissionInfo readTsPermissions) {
List<GroupPermissions> permissions = new ArrayList<>();
for (EntityGroupId egId : readPermissions.getEntityGroupIds()) {
permissions.add(new GroupPermissions(egId.getId(),
readAttrPermissions.getEntityGroupIds() != null && readAttrPermissions.getEntityGroupIds().contains(egId),
readTsPermissions.getEntityGroupIds() != null && readTsPermissions.getEntityGroupIds().contains(egId)));
}
return permissions;
}
protected static boolean checkCustomerHierarchy(Set<UUID> customers, EntityData<?> ed) {
return ed.getCustomerId() != null && customers.contains(ed.getCustomerId());
}
protected void process(Collection<EntityData<?>> entities, Consumer<EntityData<?>> processor) {
for (EntityData<?> ed : entities) {
if (matches(ed)) {
@ -122,6 +62,11 @@ public abstract class AbstractQueryProcessor<T extends EntityFilter> implements
}
}
protected static boolean checkCustomerId(UUID customerId, EntityData<?> ed) {
return customerId.equals(ed.getCustomerId()) || (ed.getEntityType() == EntityType.DASHBOARD &&
ed.getFields().getAssignedCustomerIds().contains(customerId));
}
protected boolean matches(EntityData<?> ed) {
return checkFilters(query, ed);
}

112
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,9 +16,7 @@
package org.thingsboard.server.edqs.query.processor;
import lombok.RequiredArgsConstructor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.permission.Resource;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
@ -37,7 +35,6 @@ import java.util.Queue;
import java.util.Set;
import java.util.UUID;
import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue;
public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> extends AbstractQueryProcessor<T> {
@ -76,123 +73,36 @@ public abstract class AbstractRelationQueryProcessor<T extends EntityFilter> ext
var entities = getEntitiesSet(relations);
long result = 0;
RelationQueryPermissions[] permissionsArray = buildPermissionsArray();
if (ctx.isTenantUser()) {
for (EntityData<?> ed : entities) {
var permissions = permissionsArray[ed.getEntityType().ordinal()];
if (permissions != null) {
if (permissions.isHasGroups()) {
CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(),
permissions.isReadEntity(), permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions());
if (combinedPermissions.isRead()) {
result++;
}
} else if (permissions.isReadEntity()) {
result++;
}
}
}
return entities.size();
} else {
var customerIds = repository.getAllCustomers(ctx.getCustomerId().getId());
var customerId = ctx.getCustomerId().getId();
for (EntityData<?> ed : entities) {
var permissions = permissionsArray[ed.getEntityType().ordinal()];
if (permissions != null) {
boolean isReadEntity = permissions.isReadEntity() && ed.getCustomerId() != null && customerIds.contains(ed.getCustomerId());
if (permissions.isHasGroups()) {
CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(),
isReadEntity,
permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions());
if (combinedPermissions.isRead()) {
result++;
}
} else if (isReadEntity) {
result++;
}
if (checkCustomerId(customerId, ed)) {
result++;
}
}
return result;
}
return result;
}
private List<SortableEntityData> processTenantQuery(Set<EntityData<?>> entities) {
List<SortableEntityData> result = new ArrayList<>();
RelationQueryPermissions[] permissionsArray = buildPermissionsArray();
for (EntityData<?> ed : entities) {
var permissions = permissionsArray[ed.getEntityType().ordinal()];
if (permissions != null) {
if (permissions.isHasGroups()) {
CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(),
permissions.isReadEntity(), permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions());
if (combinedPermissions.isRead()) {
SortableEntityData sortData = new SortableEntityData(ed);
sortData.setSortValue(getSortValue(ed, sortKey));
sortData.setReadAttrs(combinedPermissions.isReadAttrs());
sortData.setReadTs(combinedPermissions.isReadTs());
result.add(sortData);
}
} else if (permissions.isReadEntity()) {
result.add(toSortData(ed, permissions));
}
}
}
return result;
return entities.stream()
.map(this::toSortData)
.toList();
}
private List<SortableEntityData> processCustomerQuery(Set<EntityData<?>> entities) {
var customerIds = repository.getAllCustomers(ctx.getCustomerId().getId());
RelationQueryPermissions[] permissionsArray = buildPermissionsArray();
var customerId = ctx.getCustomerId().getId();
List<SortableEntityData> result = new ArrayList<>();
for (EntityData<?> ed : entities) {
var permissions = permissionsArray[ed.getEntityType().ordinal()];
if (permissions != null) {
boolean isReadEntity = permissions.isReadEntity() && ed.getCustomerId() != null && customerIds.contains(ed.getCustomerId());
if (permissions.isHasGroups()) {
SortableEntityData sortData = new SortableEntityData(ed);
sortData.setSortValue(getSortValue(ed, sortKey));
CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(),
isReadEntity,
permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions());
if (combinedPermissions.isRead()) {
sortData.setReadAttrs(combinedPermissions.isReadAttrs());
sortData.setReadTs(combinedPermissions.isReadTs());
result.add(sortData);
}
} else if (isReadEntity) {
result.add(toSortData(ed, permissions));
}
if (checkCustomerId(customerId, ed)) {
result.add(toSortData(ed));
}
}
return result;
}
private RelationQueryPermissions[] buildPermissionsArray() {
RelationQueryPermissions[] permissionsArray = new RelationQueryPermissions[EntityType.values().length];
var readEntityPermissionsMap = ctx.getMergedReadEntityPermissionsMap();
var readAttrPermissionsMap = ctx.getMergedReadAttrPermissionsMap();
var readTsPermissionsMap = ctx.getMergedReadTsPermissionsMap();
for (EntityType et : EntityType.values()) {
var resource = Resource.resourceFromEntityType(et);
if (resource == null) {
continue;
}
var readEntityPermissions = readEntityPermissionsMap.get(resource);
var readAttrPermissions = readAttrPermissionsMap.get(resource);
var readTsPermissions = readTsPermissionsMap.get(resource);
var groupPermissions = toGroupPermissions(readEntityPermissions, readAttrPermissions, readTsPermissions);
RelationQueryPermissions entityPermissions = RelationQueryPermissions
.builder()
.readEntity(readEntityPermissions.isHasGenericRead())
.readAttrs(readAttrPermissions.isHasGenericRead())
.readTs(readTsPermissions.isHasGenericRead())
.hasGroups(!groupPermissions.isEmpty())
.groupPermissions(groupPermissions)
.build();
permissionsArray[et.ordinal()] = entityPermissions;
}
return permissionsArray;
}
private Set<EntityData<?>> getEntitiesSet(RelationsRepo relations) {
Set<EntityData<?>> result = new HashSet<>();
Set<UUID> processed = new HashSet<>();

53
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,15 +20,9 @@ import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.edqs.data.CustomerData;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.data.EntityGroupData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Consumer;
@ -42,47 +36,10 @@ public abstract class AbstractSimpleQueryProcessor<T extends EntityFilter> exten
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
var customers = repository.getEntityMap(EntityType.CUSTOMER);
for (UUID cId : repository.getAllCustomers(customerId)) {
var customerData = (CustomerData) customers.get(cId);
if (customerData != null) {
process(customerData.getEntities(entityType), processor);
}
}
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
var genericReadResults = processCustomerGenericRead(customerId, readAttrPermissions, readTsPermissions);
Map<UUID, SortableEntityData> mergedResult = new HashMap<>(genericReadResults.size());
for (SortableEntityData sd : genericReadResults) {
mergedResult.put(sd.getId(), sd);
}
for (GroupPermissions permissions : groupPermissions) {
EntityGroupData egData = repository.getEntityGroup(permissions.groupId);
for (EntityData<?> ed : egData.getEntities()) {
SortableEntityData alreadyAdded = mergedResult.get(ed.getId());
if (alreadyAdded != null) {
alreadyAdded.setReadAttrs(alreadyAdded.isReadAttrs() || permissions.readAttrs);
alreadyAdded.setReadTs(alreadyAdded.isReadTs() || permissions.readTs);
} else {
if (matches(ed)) {
SortableEntityData sortData = toSortData(ed, permissions);
mergedResult.put(ed.getId(), sortData);
}
}
}
}
return new ArrayList<>(mergedResult.values());
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
for (GroupPermissions groupPermission : groupPermissions) {
EntityGroupData egData = repository.getEntityGroup(groupPermission.groupId);
process(egData.getEntities(), processor);
protected void processCustomerQuery(UUID customerId, Consumer<EntityData<?>> processor) {
var customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId);
if (customerData != null) {
process(customerData.getEntities(entityType), processor);
}
}

116
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -15,6 +15,7 @@
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.edqs.data.EntityData;
@ -23,7 +24,6 @@ import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
@ -37,133 +37,45 @@ public abstract class AbstractSingleEntityTypeQueryProcessor<T extends EntityFil
@Override
public List<SortableEntityData> processQuery() {
var readPermissions = ctx.getMergedReadPermissionsByEntityType();
if (readPermissions == null) {
return Collections.emptyList();
}
var readAttrPermissions = ctx.getMergedReadAttrPermissionsByEntityType();
var readTsPermissions = ctx.getMergedReadTsPermissionsByEntityType();
boolean hasGenericRead = readPermissions.isHasGenericRead();
boolean hasGroups = readPermissions.getEntityGroupIds() != null && !readPermissions.getEntityGroupIds().isEmpty();
if (!hasGenericRead && !hasGroups) {
return Collections.emptyList();
}
boolean hasGenericAttrRead = readAttrPermissions.isHasGenericRead();
boolean hasGenericTsRead = readTsPermissions.isHasGenericRead();
if (hasGenericRead) {
if (ctx.isTenantUser()) {
if (hasGroups && (!hasGenericAttrRead || !hasGenericTsRead)) {
return processTenantGenericReadWithGroups(hasGenericAttrRead, hasGenericTsRead,
toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions));
} else {
return processTenantGenericRead(hasGenericAttrRead, hasGenericTsRead);
}
} else {
if (hasGroups) {
return processCustomerGenericReadWithGroups(ctx.getCustomerId().getId(), hasGenericAttrRead, hasGenericTsRead,
toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions));
} else {
return processCustomerGenericRead(ctx.getCustomerId().getId(), hasGenericAttrRead, hasGenericTsRead);
}
}
if (ctx.isTenantUser()) {
return processTenantQuery();
} else {
return processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions));
return processCustomerQuery(ctx.getCustomerId().getId());
}
}
@Override
public long count() { // TODO: get rid of the duplicates
var readPermissions = ctx.getMergedReadPermissionsByEntityType();
if (readPermissions == null) {
return 0;
}
var readAttrPermissions = ctx.getMergedReadAttrPermissionsByEntityType();
var readTsPermissions = ctx.getMergedReadTsPermissionsByEntityType();
boolean hasGenericRead = readPermissions.isHasGenericRead();
boolean hasGroups = readPermissions.getEntityGroupIds() != null && !readPermissions.getEntityGroupIds().isEmpty();
if (!hasGenericRead && !hasGroups && !ctx.isIgnorePermissionCheck()) {
return 0;
}
public long count() {
AtomicLong result = new AtomicLong();
Consumer<EntityData<?>> counter = ed -> result.incrementAndGet();
if (ctx.isIgnorePermissionCheck()) {
processAll(counter);
} else if (ctx.isTenantUser()) {
if (hasGenericRead) {
processAll(counter);
} else {
processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions), counter);
}
processAll(counter);
} else {
if (hasGenericRead) {
if (hasGroups) {
result.addAndGet(processCustomerGenericReadWithGroups(ctx.getCustomerId().getId(), readAttrPermissions.isHasGenericRead(), readTsPermissions.isHasGenericRead(),
toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)).size()); // FIXME: not efficient
} else {
processCustomerGenericRead(ctx.getCustomerId().getId(), counter);
}
} else {
processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions), counter);
}
processCustomerQuery(ctx.getCustomerId().getId(), counter);
}
return result.get();
}
protected List<SortableEntityData> processTenantGenericRead(boolean readAttrPermissions,
boolean readTsPermissions) {
List<SortableEntityData> result = new ArrayList<>(getProbableResultSize());
processAll(ed -> {
result.add(toSortData(ed, readAttrPermissions, readTsPermissions));
});
return result;
}
protected List<SortableEntityData> processCustomerGenericRead(UUID customerId,
boolean readAttrPermissions,
boolean readTsPermissions) {
List<SortableEntityData> result = new ArrayList<>(getProbableResultSize());
processCustomerGenericRead(customerId, ed -> {
result.add(toSortData(ed, readAttrPermissions, readTsPermissions));
});
return result;
}
protected abstract void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor);
protected List<SortableEntityData> processTenantGenericReadWithGroups(boolean readAttrPermissions,
boolean readTsPermissions,
List<GroupPermissions> groupPermissions) {
protected List<SortableEntityData> processTenantQuery() {
List<SortableEntityData> result = new ArrayList<>(getProbableResultSize());
processAll(ed -> {
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), true, readAttrPermissions, readTsPermissions, groupPermissions);
SortableEntityData sortData = toSortData(ed, permissions);
result.add(sortData);
result.add(toSortData(ed));
});
return result;
}
protected abstract List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId,
boolean readAttrPermissions,
boolean readTsPermissions,
List<GroupPermissions> groupPermissions);
protected List<SortableEntityData> processGroupsOnly(List<GroupPermissions> groupPermissions) {
protected List<SortableEntityData> processCustomerQuery(UUID customerId) {
List<SortableEntityData> result = new ArrayList<>(getProbableResultSize());
processGroupsOnly(groupPermissions, ed -> {
SortableEntityData sortData = toSortDataGroupsOnly(ed, groupPermissions);
if (sortData != null) {
result.add(sortData);
}
processCustomerQuery(customerId, ed -> {
result.add(toSortData(ed));
});
return result;
}
protected abstract void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor);
protected abstract void processCustomerQuery(UUID customerId, Consumer<EntityData<?>> processor);
protected abstract void processAll(Consumer<EntityData<?>> processor);

35
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,12 +22,8 @@ import org.thingsboard.server.common.data.query.ApiUsageStateFilter;
import org.thingsboard.server.edqs.data.CustomerData;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
@ -38,36 +34,13 @@ public class ApiUsageStateQueryProcessor extends AbstractSingleEntityTypeQueryPr
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
protected void processCustomerQuery(UUID customerId, Consumer<EntityData<?>> processor) {
CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId);
process(customerData.getEntities(EntityType.API_USAGE_STATE), processor);
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId);
Collection<EntityData<?>> entities = customerData.getEntities(EntityType.API_USAGE_STATE);
EntityData<?> ed = entities.iterator().next();
if (entities.isEmpty() || !matches(ed)) {
return Collections.emptyList();
} else {
boolean genericRead = customerId.equals(ed.getCustomerId());
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions);
if (permissions.isRead()) {
SortableEntityData sortData = toSortData(customerData, permissions);
return Collections.singletonList(sortData);
} else {
return Collections.emptyList();
}
if (customerData != null) {
process(customerData.getEntities(EntityType.API_USAGE_STATE), processor);
}
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
processAll(processor);
}
@Override
protected void processAll(Consumer<EntityData<?>> processor) {
process(repository.getEntitySet(EntityType.API_USAGE_STATE), processor);

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

25
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java

@ -1,25 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import lombok.Data;
@Data
public class CombinedPermissions implements Permissions {
private final boolean read;
private final boolean readAttrs;
private final boolean readTs;
}

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

121
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java

@ -1,121 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntitiesByGroupNameFilter;
import org.thingsboard.server.edqs.data.CustomerData;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import org.thingsboard.server.edqs.util.RepositoryUtils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import static org.thingsboard.server.common.data.EntityType.ENTITY_GROUP;
import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue;
public class EntitiesByGroupNameQueryProcessor extends AbstractSingleEntityTypeQueryProcessor<EntitiesByGroupNameFilter> {
private final String groupType;
private final UUID ownerId;
private final Pattern pattern;
public EntitiesByGroupNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) {
super(repo, ctx, query, (EntitiesByGroupNameFilter) query.getEntityFilter());
this.groupType = filter.getGroupType().name();
this.ownerId = filter.getOwnerId() != null ? filter.getOwnerId().getId() : null;
this.pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityGroupNameFilter());
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
var customers = repository.getEntityMap(EntityType.CUSTOMER);
for (UUID cId : repository.getAllCustomers(customerId)) {
var customerData = (CustomerData) customers.get(cId);
if (customerData != null) {
process(customerData.getEntities(ENTITY_GROUP), processor);
}
}
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
List<SortableEntityData> result = new ArrayList<>(getProbableResultSize());
var customers = repository.getAllCustomers(customerId);
processAll(ed -> {
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions);
if (permissions.isRead()) {
SortableEntityData sortData = new SortableEntityData(ed);
sortData.setSortValue(getSortValue(ed, sortKey));
sortData.setReadAttrs(permissions.isReadAttrs());
sortData.setReadTs(permissions.isReadTs());
result.add(sortData);
}
});
return result;
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
Collection<EntityData<?>> entities = new HashSet<>(getProbableResultSize());
for (GroupPermissions groupPermission : groupPermissions) {
entities.add(repository.getEntityGroup(groupPermission.groupId));
}
process(entities, processor);
}
@Override
protected void processAll(Consumer<EntityData<?>> processor) {
process(repository.getEntitySet(ENTITY_GROUP), processor);
}
@Override
protected void process(Collection<EntityData<?>> entities, Consumer<EntityData<?>> processor) {
for (EntityData<?> ed : entities) {
if (matches(ed)) {
Collection<EntityData<?>> groupEntities = repository.getEntityGroup(ed.getId()).getEntities();
for (EntityData<?> groupEntity : groupEntities) {
processor.accept(groupEntity);
}
return;
}
}
}
@Override
protected boolean matches(EntityData ed) {
EntityGroupFields fields = (EntityGroupFields)ed.getFields();
return super.matches(ed) && groupType.equals(fields.getType())
&& (pattern == null || pattern.matcher(fields.getName()).matches())
&& (ownerId == null || ownerId.equals(fields.getOwnerId()));
}
@Override
protected int getProbableResultSize() {
return 1024;
}
}

96
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java

@ -1,96 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityGroupFilter;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.data.EntityGroupData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue;
public class EntitiesByGroupQueryProcessor extends AbstractSingleEntityTypeQueryProcessor<EntityGroupFilter> {
private final String groupType;
private final UUID groupId;
public EntitiesByGroupQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) {
super(repo, ctx, query, (EntityGroupFilter) query.getEntityFilter());
groupId = UUID.fromString(filter.getEntityGroup());
groupType = filter.getGroupType().name();
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
var customers = repository.getAllCustomers(customerId);
processAll(ed -> {
if (checkCustomerHierarchy(customers, ed)) {
processor.accept(ed);
}
});
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
List<SortableEntityData> result = new ArrayList<>(getProbableResultSize());
var customers = repository.getAllCustomers(customerId);
processAll(ed -> {
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions);
if (permissions.isRead()) {
SortableEntityData sortData = new SortableEntityData(ed);
sortData.setSortValue(getSortValue(ed, sortKey));
sortData.setReadAttrs(permissions.isReadAttrs());
sortData.setReadTs(permissions.isReadTs());
result.add(sortData);
}
});
return result;
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
processAll(processor);
}
@Override
protected void processAll(Consumer<EntityData<?>> processor) {
EntityGroupData entityGroup = repository.getEntityGroup(groupId);
if (matches(entityGroup)) {
for (EntityData<?> ed : entityGroup.getEntities()) {
processor.accept(ed);
}
}
}
@Override
protected boolean matches(EntityData ed) {
return super.matches(ed) && groupType.equals(ed.getFields().getType());
}
@Override
protected int getProbableResultSize() {
return 1024;
}
}

84
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java

@ -1,84 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityGroupListFilter;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.data.EntityGroupData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.stream.Collectors;
public class EntityGroupListQueryProcessor extends AbstractEntityGroupQueryProcessor<EntityGroupListFilter> {
private final String groupType;
private final Set<UUID> groupIds;
public EntityGroupListQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) {
super(repo, ctx, query, (EntityGroupListFilter) query.getEntityFilter());
this.groupType = filter.getGroupType().name();
this.groupIds = filter.getEntityGroupList().stream().map(UUID::fromString).collect(Collectors.toSet());
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
var customers = repository.getAllCustomers(customerId);
processAll(ed -> {
if (checkCustomerHierarchy(customers, ed)) {
processor.accept(ed);
}
});
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
Set<UUID> allowedGroupIds = groupPermissions.stream().map(GroupPermissions::getGroupId)
.filter(this.groupIds::contains).collect(Collectors.toSet());
checkGroupIds(allowedGroupIds, processor);
}
@Override
protected void processAll(Consumer<EntityData<?>> processor) {
checkGroupIds(groupIds, processor);
}
@Override
protected int getProbableResultSize() {
return groupIds.size();
}
@Override
protected boolean matches(EntityData ed) {
return super.matches(ed) && groupType.equals(ed.getFields().getType());
}
private void checkGroupIds(Set<UUID> allowedGroupIds, Consumer<EntityData<?>> processor) {
for (UUID groupId : allowedGroupIds) {
EntityGroupData entityGroup = repository.getEntityGroup(groupId);
if (matches(entityGroup)) {
processor.accept(entityGroup);
}
}
}
}

83
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java

@ -1,83 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityGroupNameFilter;
import org.thingsboard.server.edqs.data.CustomerData;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.data.EntityGroupData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.repo.TenantRepo;
import org.thingsboard.server.edqs.util.RepositoryUtils;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import static org.thingsboard.server.common.data.EntityType.ENTITY_GROUP;
public class EntityGroupNameQueryProcessor extends AbstractEntityGroupQueryProcessor<EntityGroupNameFilter> {
private final String groupType;
private final Pattern groupNamePattern;
public EntityGroupNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) {
super(repo, ctx, query, (EntityGroupNameFilter) query.getEntityFilter());
this.groupType = filter.getGroupType().name();
this.groupNamePattern = RepositoryUtils.toSqlLikePattern(filter.getEntityGroupNameFilter());
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
var customers = repository.getEntityMap(EntityType.CUSTOMER);
for (UUID cId : repository.getAllCustomers(customerId)) {
var customerData = (CustomerData) customers.get(cId);
if (customerData != null) {
process(customerData.getEntities(ENTITY_GROUP), processor);
}
}
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
for (GroupPermissions groupPermission : groupPermissions) {
EntityGroupData entityGroup = repository.getEntityGroup(groupPermission.groupId);
if (matches(entityGroup)) {
processor.accept(entityGroup);
}
}
}
@Override
protected void processAll(Consumer<EntityData<?>> processor) {
process(repository.getEntitySet(ENTITY_GROUP), processor);
}
@Override
protected boolean matches(EntityData ed) {
return super.matches(ed) && (groupNamePattern == null || groupNamePattern.matcher(ed.getFields().getName()).matches())
&& groupType.equals(ed.getFields().getType());
}
@Override
protected int getProbableResultSize() {
return 1024;
}
}

34
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,18 +20,13 @@ import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityListFilter;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue;
public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProcessor<EntityListFilter> {
private final EntityType entityType;
@ -44,37 +39,14 @@ public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProce
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
var customers = repository.getAllCustomers(customerId);
protected void processCustomerQuery(UUID customerId, Consumer<EntityData<?>> processor) {
processAll(ed -> {
if (checkCustomerHierarchy(customers, ed)) {
if (checkCustomerId(customerId, ed)) {
processor.accept(ed);
}
});
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
List<SortableEntityData> result = new ArrayList<>(getProbableResultSize());
var customers = repository.getAllCustomers(customerId);
processAll(ed -> {
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions);
if (permissions.isRead()) {
SortableEntityData sortData = new SortableEntityData(ed);
sortData.setSortValue(getSortValue(ed, sortKey));
sortData.setReadAttrs(permissions.isReadAttrs());
sortData.setReadTs(permissions.isReadTs());
result.add(sortData);
}
});
return result;
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
processAll(processor);
}
@Override
protected void processAll(Consumer<EntityData<?>> processor) {
var map = repository.getEntityMap(entityType);

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

8
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -32,17 +32,11 @@ public class EntityQueryProcessorFactory {
case ENTITY_VIEW_TYPE -> new EntityViewTypeQueryProcessor(repo, ctx, query);
case EDGE_TYPE -> new EdgeTypeQueryProcessor(repo, ctx, query);
case RELATIONS_QUERY -> new RelationQueryProcessor(repo, ctx, query);
case ENTITY_GROUP -> new EntitiesByGroupQueryProcessor(repo, ctx, query);
case ENTITY_GROUP_LIST -> new EntityGroupListQueryProcessor(repo, ctx, query);
case ENTITY_GROUP_NAME -> new EntityGroupNameQueryProcessor(repo, ctx, query);
case ENTITIES_BY_GROUP_NAME -> new EntitiesByGroupNameQueryProcessor(repo, ctx, query);
case STATE_ENTITY_OWNER -> new StateEntityOwnerQueryProcessor(repo, ctx, query);
case API_USAGE_STATE -> new ApiUsageStateQueryProcessor(repo, ctx, query);
case ASSET_SEARCH_QUERY -> new AssetSearchQueryProcessor(repo, ctx, query);
case DEVICE_SEARCH_QUERY -> new DeviceSearchQueryProcessor(repo, ctx, query);
case ENTITY_VIEW_SEARCH_QUERY -> new EntityViewSearchQueryProcessor(repo, ctx, query);
case EDGE_SEARCH_QUERY -> new EdgeTypeSearchQueryProcessor(repo, ctx, query);
case SCHEDULER_EVENT -> new SchedulerEventQueryProcessor(repo, ctx, query);
default -> throw new RuntimeException("Not Implemented!");
};
}

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

29
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java

@ -1,29 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import lombok.Data;
import java.util.UUID;
@Data
public class GroupPermissions implements Permissions {
protected final UUID groupId;
protected final boolean readAttrs;
protected final boolean readTs;
}

24
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java

@ -1,24 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
public interface Permissions {
boolean isReadAttrs();
boolean isReadTs();
}

33
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java

@ -1,33 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import lombok.Builder;
import lombok.Getter;
import java.util.List;
@Getter
@Builder
public class RelationQueryPermissions implements Permissions {
private final boolean readEntity;
private final boolean readAttrs;
private final boolean readTs;
private final boolean hasGroups;
private final List<GroupPermissions> groupPermissions;
}

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

37
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java

@ -1,37 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.SchedulerEventFilter;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.repo.TenantRepo;
public class SchedulerEventQueryProcessor extends AbstractSimpleQueryProcessor<SchedulerEventFilter> {
public SchedulerEventQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) {
super(repo, ctx, query, (SchedulerEventFilter) query.getEntityFilter(), EntityType.SCHEDULER_EVENT);
}
@Override
protected boolean matches(EntityData<?> ed) {
return super.matches(ed) && (filter.getEventType() == null || filter.getEventType().equals(ed.getFields().getType()))
&& (filter.getOriginator() == null || filter.getOriginator().equals(ed.getFields().getOriginatorId()));
}
}

36
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,11 +20,8 @@ import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
@ -40,35 +37,12 @@ public class SingleEntityQueryProcessor extends AbstractSingleEntityTypeQueryPro
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
EntityData ed = repository.getEntityMap(entityType).get(entityId);
if (ed != null && ed.getCustomerId() != null && matches(ed)) {
if (customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId())) {
protected void processCustomerQuery(UUID customerId, Consumer<EntityData<?>> processor) {
processAll(ed -> {
if (checkCustomerId(customerId, ed)) {
processor.accept(ed);
}
}
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
EntityData ed = repository.getEntityMap(entityType).get(entityId);
if (!matches(ed)) {
return Collections.emptyList();
} else {
boolean genericRead = customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId());
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions);
if (permissions.isRead()) {
SortableEntityData sortData = toSortData(ed, permissions);
return Collections.singletonList(sortData);
} else {
return Collections.emptyList();
}
}
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
processAll(processor);
});
}
@Override

91
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java

@ -1,91 +0,0 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.query.processor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.StateEntityOwnerFilter;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.query.EdqsQuery;
import org.thingsboard.server.edqs.query.SortableEntityData;
import org.thingsboard.server.edqs.repo.TenantRepo;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.function.Consumer;
public class StateEntityOwnerQueryProcessor extends AbstractSingleEntityTypeQueryProcessor<StateEntityOwnerFilter> {
private final EntityId entityId;
public StateEntityOwnerQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) {
super(repo, ctx, query, (StateEntityOwnerFilter) query.getEntityFilter());
this.entityId = filter.getSingleEntity();
}
@Override
protected void processCustomerGenericRead(UUID customerId, Consumer<EntityData<?>> processor) {
EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId());
if (ed != null && ed.getCustomerId() != null && matches(ed)) {
if (customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId())) {
processor.accept(ed);
}
}
}
@Override
protected List<SortableEntityData> processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List<GroupPermissions> groupPermissions) {
EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId());
if (!matches(ed)) {
return Collections.emptyList();
} else {
boolean genericRead = customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId());
CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions);
if (permissions.isRead()) {
SortableEntityData sortData = toSortData(ed, permissions);
return Collections.singletonList(sortData);
} else {
return Collections.emptyList();
}
}
}
@Override
protected void processGroupsOnly(List<GroupPermissions> groupPermissions, Consumer<EntityData<?>> processor) {
processAll(processor);
}
@Override
protected void processAll(Consumer<EntityData<?>> processor) {
EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId());
if (ed != null) {
if (ed.getCustomerId() != null) {
processor.accept(repository.getEntityMap(EntityType.CUSTOMER).get(ed.getCustomerId()));
} else {
processor.accept(repository.getEntityMap(EntityType.TENANT).get(repository.getTenantId().getId()));
}
}
}
@Override
protected int getProbableResultSize() {
return 1;
}
}

7
common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.edqs.query.QueryResult;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.permission.MergedUserPermissions;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityDataQuery;
@ -33,9 +32,9 @@ public interface EdqRepository {
@Deprecated
default void addOrUpdate(TenantId tenantId, Object object) {}
long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery query, boolean ignorePermissionCheck);
long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck);
PageData<QueryResult> findEntityDataByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityDataQuery query, boolean ignorePermissionCheck);
PageData<QueryResult> findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck);
void clearIf(Predicate<TenantId> predicate);

22
common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,12 +18,13 @@ package org.thingsboard.server.edqs.repo;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsEvent;
import org.thingsboard.server.common.data.edqs.EdqsEventType;
import org.thingsboard.server.common.data.edqs.query.QueryResult;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.permission.MergedUserPermissions;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.edqs.stats.EdqsStatsService;
@ -49,19 +50,24 @@ public class InMemoryEdqRepository implements EdqRepository {
@Override
public void processEvent(EdqsEvent event) {
get(event.getTenantId()).processEvent(event);
if (event.getEventType() == EdqsEventType.DELETED && event.getObjectType() == ObjectType.TENANT) {
log.info("Deleting tenant repo: {}", event);
repos.remove(event.getTenantId());
} else {
get(event.getTenantId()).processEvent(event);
}
}
@Override
public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery query, boolean ignorePermissionCheck) {
public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) {
long startNs = System.nanoTime();
long result = 0;
if (TenantId.SYS_TENANT_ID.equals(tenantId)) {
for (TenantRepo repo : repos.values()) {
result += repo.countEntitiesByQuery(customerId, userPermissions, query, ignorePermissionCheck);
result += repo.countEntitiesByQuery(customerId, query, ignorePermissionCheck);
}
} else {
result = get(tenantId).countEntitiesByQuery(customerId, userPermissions, query, ignorePermissionCheck);
result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck);
}
double timingMs = (double) (System.nanoTime() - startNs) / 1000_000;
log.info("countEntitiesByQuery: {} ms", timingMs);
@ -70,9 +76,9 @@ public class InMemoryEdqRepository implements EdqRepository {
@Override
public PageData<QueryResult> findEntityDataByQuery(TenantId tenantId, CustomerId customerId,
MergedUserPermissions userPermissions, EntityDataQuery query, boolean ignorePermissionCheck) {
EntityDataQuery query, boolean ignorePermissionCheck) {
long startNs = System.nanoTime();
var result = get(tenantId).findEntityDataByQuery(customerId, userPermissions, query, ignorePermissionCheck);
var result = get(tenantId).findEntityDataByQuery(customerId, query, ignorePermissionCheck);
double timingMs = (double) (System.nanoTime() - startNs) / 1000_000;
log.info("findEntityDataByQuery: {} ms", timingMs);
return result;

2
common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

173
common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -25,24 +25,19 @@ import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.LatestTsKv;
import org.thingsboard.server.common.data.edqs.fields.AssetFields;
import org.thingsboard.server.common.data.edqs.fields.CustomerFields;
import org.thingsboard.server.common.data.edqs.fields.EntityFields;
import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields;
import org.thingsboard.server.common.data.edqs.query.QueryResult;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.permission.MergedUserPermissions;
import org.thingsboard.server.common.data.permission.QueryContext;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityFilter;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.common.data.query.StateEntityOwnerFilter;
import org.thingsboard.server.common.data.query.TsValue;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
@ -51,7 +46,6 @@ import org.thingsboard.server.edqs.data.AssetData;
import org.thingsboard.server.edqs.data.CustomerData;
import org.thingsboard.server.edqs.data.DeviceData;
import org.thingsboard.server.edqs.data.EntityData;
import org.thingsboard.server.edqs.data.EntityGroupData;
import org.thingsboard.server.edqs.data.EntityProfileData;
import org.thingsboard.server.edqs.data.GenericData;
import org.thingsboard.server.edqs.data.RelationsRepo;
@ -76,12 +70,9 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
@ -95,7 +86,6 @@ import java.util.stream.Collectors;
import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_ASC;
import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_DESC;
import static org.thingsboard.server.edqs.util.RepositoryUtils.SYS_ADMIN_PERMISSIONS;
import static org.thingsboard.server.edqs.util.RepositoryUtils.resolveEntityType;
@Slf4j
@ -108,8 +98,6 @@ public class TenantRepo {
private final ConcurrentMap<EntityType, Set<EntityData<?>>> entitySetByType = new ConcurrentHashMap<>();
private final ConcurrentMap<EntityType, ConcurrentMap<UUID, EntityData<?>>> entityMapByType = new ConcurrentHashMap<>();
private final ConcurrentMap<UUID, Set<UUID>> customersHierarchy = new ConcurrentHashMap<>();
private final ConcurrentMap<UUID, EntityGroupData> entityGroups = new ConcurrentHashMap<>();
private final ConcurrentMap<RelationTypeGroup, RelationsRepo> relations = new ConcurrentHashMap<>();
private final Lock entityUpdateLock = new ReentrantLock();
@ -167,10 +155,10 @@ public class TenantRepo {
if (added) {
edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED));
}
} else if (RelationTypeGroup.FROM_ENTITY_GROUP.equals(entity.getTypeGroup())) {
var eg = getEntityGroup(entity.getFrom().getId());
if (eg != null) {
eg.addOrUpdate(getOrCreate(entity.getTo()));
} else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) {
if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) {
((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entity.getFrom().getId(), CustomerData::new))
.addOrUpdate(getEntityMap(EntityType.DASHBOARD).get(entity.getTo().getId()));
}
}
} finally {
@ -187,10 +175,10 @@ public class TenantRepo {
edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.DELETED));
}
}
} else if (RelationTypeGroup.FROM_ENTITY_GROUP.equals(entityRelation.getTypeGroup())) {
var eg = getEntityGroup(entityRelation.getFrom().getId());
if (eg != null) {
eg.remove(entityRelation.getTo().getId());
} else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) {
if (EntityRelation.CONTAINS_TYPE.equals(entityRelation.getType()) && entityRelation.getFrom().getEntityType() == EntityType.CUSTOMER) {
((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entityRelation.getFrom().getId(), CustomerData::new))
.remove(getEntityMap(EntityType.DASHBOARD).get(entityRelation.getTo().getId()));
}
}
}
@ -207,46 +195,19 @@ public class TenantRepo {
processFields(fields);
entityData.setFields(entity.getFields());
switch (entity.getType()) {
case ENTITY_GROUP -> {
EntityGroupFields entityGroupFields = (EntityGroupFields) fields;
UUID ownerId = entityGroupFields.getOwnerId();
if (EntityType.CUSTOMER.equals(entityGroupFields.getOwnerType())) {
entityData.setCustomerId(ownerId);
((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(ownerId, CustomerData::new)).addOrUpdate(entityData);
UUID newCustomerId = fields.getCustomerId();
UUID oldCustomerId = entityData.getCustomerId();
entityData.setCustomerId(newCustomerId);
if (entityIdMismatch(oldCustomerId, newCustomerId)) {
if (oldCustomerId != null) {
CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId);
if (old != null) {
old.remove(entityData);
}
entityGroups.put(fields.getId(), (EntityGroupData) entityData);
}
case CUSTOMER -> {
CustomerFields customerFields = (CustomerFields) fields;
UUID newParentId = customerFields.getCustomerId(); // for customer, customerId is parentCustomerId
UUID oldParentId = entityData.getCustomerId();
entityData.setCustomerId(newParentId);
if (entityIdMismatch(oldParentId, newParentId)) {
if (oldParentId != null) {
customersHierarchy.computeIfAbsent(oldParentId, id -> new HashSet<>()).remove(entityData.getId());
}
if (newParentId != null) {
customersHierarchy.computeIfAbsent(newParentId, id -> new HashSet<>()).add(entityData.getId());
}
}
}
default -> {
UUID newCustomerId = fields.getCustomerId();
UUID oldCustomerId = entityData.getCustomerId();
entityData.setCustomerId(newCustomerId);
if (entityIdMismatch(oldCustomerId, newCustomerId)) {
if (oldCustomerId != null) {
CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId);
if (old != null) {
old.remove(entityData);
}
}
if (newCustomerId != null) {
CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new);
newData.addOrUpdate(entityData);
}
}
if (newCustomerId != null) {
CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new);
newData.addOrUpdate(entityData);
}
}
} finally {
@ -263,13 +224,12 @@ public class TenantRepo {
if (removed != null) {
getEntitySet(entityType).remove(removed);
edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED));
}
switch (entityType) {
case ENTITY_GROUP -> {
entityGroups.remove(entityId);
}
case CUSTOMER -> {
customersHierarchy.remove(entityId);
UUID customerId = removed.getCustomerId();
if (customerId != null) {
CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId);
if (customerData != null) {
customerData.remove(removed);
}
}
}
} finally {
@ -387,7 +347,6 @@ public class TenantRepo {
case DEVICE_PROFILE, ASSET_PROFILE -> new EntityProfileData(id, entityType);
case CUSTOMER -> new CustomerData(id);
case TENANT -> new TenantData(id);
case ENTITY_GROUP -> new EntityGroupData(id);
case API_USAGE_STATE -> new ApiUsageStateData(id);
default -> new GenericData(entityType, id);
};
@ -407,11 +366,10 @@ public class TenantRepo {
return entitySetByType.computeIfAbsent(entityType, et -> new ConcurrentSkipListSet<>(CREATED_TIME_AND_ID_DESC_COMPARATOR));
}
public PageData<QueryResult> findEntityDataByQuery(CustomerId customerId, MergedUserPermissions userPermissions,
EntityDataQuery oldQuery, boolean ignorePermissionCheck) {
public PageData<QueryResult> findEntityDataByQuery(CustomerId customerId, EntityDataQuery oldQuery, boolean ignorePermissionCheck) {
EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery);
log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query);
QueryContext ctx = buildContext(customerId, userPermissions, query.getEntityFilter(), ignorePermissionCheck);
QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck);
if (ctx == null) {
return PageData.emptyPageData();
}
@ -419,10 +377,10 @@ public class TenantRepo {
return sortAndConvert(query, queryProcessor.processQuery(), ctx);
}
public long countEntitiesByQuery(CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery oldQuery, boolean ignorePermissionCheck) {
public long countEntitiesByQuery(CustomerId customerId, EntityCountQuery oldQuery, boolean ignorePermissionCheck) {
EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery);
log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query);
QueryContext ctx = buildContext(customerId, userPermissions, query.getEntityFilter(), ignorePermissionCheck);
QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck);
if (ctx == null) {
return 0;
}
@ -485,85 +443,22 @@ public class TenantRepo {
latest.computeIfAbsent(key.type(), t -> new HashMap<>()).put(KeyDictionary.get(key.keyId()), v);
}
results.add(new QueryResult(entityData.getEntityId(), entityData.isReadAttrs(), entityData.isReadTs(), latest));
results.add(new QueryResult(entityData.getEntityId(), latest));
}
return results;
}
private QueryContext buildContext(CustomerId customerId, MergedUserPermissions userPermissions, EntityFilter filter, boolean ignorePermissionCheck) {
QueryContext queryContext;
if (TenantId.SYS_TENANT_ID.equals(tenantId)) {
queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), SYS_ADMIN_PERMISSIONS, filter, ignorePermissionCheck);
} else {
switch (filter.getType()) {
case STATE_ENTITY_OWNER:
var singleEntity = ((StateEntityOwnerFilter) filter).getSingleEntity();
EntityData ed = getEntityMap(singleEntity.getEntityType()).get(singleEntity.getId());
if (ed != null) {
EntityId owner = ed.getCustomerId() != null ? new CustomerId(ed.getCustomerId()) : tenantId;
queryContext = new QueryContext(tenantId, customerId, owner.getEntityType(), userPermissions, filter, owner, ignorePermissionCheck);
} else {
return null;
}
break;
case SINGLE_ENTITY:
SingleEntityFilter seFilter = (SingleEntityFilter) filter;
EntityId entityId = seFilter.getSingleEntity();
if (entityId != null && entityId.getEntityType().equals(EntityType.ENTITY_GROUP)) {
EntityGroupData entityGroupData = entityGroups.get(entityId.getId());
if (entityGroupData != null) {
queryContext = new QueryContext(tenantId, customerId, EntityType.ENTITY_GROUP, userPermissions, filter, entityGroupData.getEntityType(), ignorePermissionCheck);
} else {
return null;
}
} else {
queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), userPermissions, filter, ignorePermissionCheck);
}
break;
default:
queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), userPermissions, filter, ignorePermissionCheck);
}
}
return queryContext;
private QueryContext buildContext(CustomerId customerId, EntityFilter filter, boolean ignorePermissionCheck) {
return new QueryContext(tenantId, customerId, resolveEntityType(filter), ignorePermissionCheck);
}
public TenantId getTenantId() {
return tenantId;
}
public Set<UUID> getAllCustomers(UUID customerId) {
Set<UUID> result = new HashSet<>();
Queue<UUID> queue = new LinkedList<>();
if (customerId != null) {
queue.add(customerId);
}
while (!queue.isEmpty()) {
UUID current = queue.poll();
if (!result.contains(current)) {
result.add(current);
Set<UUID> children = customersHierarchy.get(current);
if (children != null) {
queue.addAll(children);
}
}
}
return result;
}
public boolean contains(UUID entityGroupID, UUID entityId) {
var groupData = entityGroups.get(entityGroupID);
return groupData != null && groupData.getEntity(entityId) != null;
}
public EntityGroupData getEntityGroup(UUID groupId) {
return entityGroups.get(groupId);
}
public RelationsRepo getRelations(RelationTypeGroup relationTypeGroup) {
return relations.get(relationTypeGroup);
return relations.computeIfAbsent(relationTypeGroup, type -> new RelationsRepo());
}
public String getOwnerName(EntityId ownerId) {

2
common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java

@ -1,5 +1,5 @@
/**
* Copyright © 2016-2024 ThingsBoard, Inc.
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save