EDQS (#3196)

* Experiments with CSV

* CSV Loader v1

* EDQ tests

* Volatile variables instead of final

* Improvements

* updated loader with new entities

* Fix double memory usage issue

* Basic data structures and load

* Minor improvements

* Snappy + Large String reuse

* added EntityFields classes for each entity

* Basic implementation

* Minor improvements to KeyFilters

* implemented RepositoryUtils.checkKeyFilters

* Generic query implementation

* New structure

* Refactoring and few processors implementation

* extended DeviceData with shared/client attributes and device profile

* Minor refactoring of attribute scopes

* DeviceTypeFilter support

* Strong types of fields for each entity data class

* DeviceType and AssetType filters

* EntityView and Edge queries

* Relations Query

* Relation Query Implementation

* Update EDQS module version

* Sync with EDQS via Kafka

* EDQS: major refactoring

* EDQS API requests via Kafka

* EDQS: full sync with the database

* Refactoring for EDQS sync

* EDQS: major refactoring and new features

* EDQS refactoring, count query support, fix tests

* EDQS: refactoring for query processors

* Fix EDQS pom version

* Cleanup edqs.yml

* EDQS: tenant partitioning strategy; refactoring

* EDQS: latest events queue

* EDQS: support for monolith setup; RocksDB; other improvements

* EDQS: merge sync and events topics, introduce state topic

* EDQS: dynamic repartitioning

* implemented entity data query filters for edqs

* EdqsEntityQueryControllerTest - use in-memory queue

* edqs-filter fixes, added test

* EDQS: blob entity support

* EdqsEntityQueryControllerTest - use in-memory queue

* Use DummyEdqsService when disabled

* Fixes for EDQS

* Refactoring for EDQS tests

* Fix edqs requests partitioning

* EDQS: Fix for attributes handling

* Fix attributes saving in EntityServiceTest

* EDQS: refactoring, fixes

* Minor refactoring for query processor

* added ownerName/ownerType support

* fixed relation query processor

* fixed EntityServiceTest

* refactoring

* added support for parentId for relation query result

* Get rid of EntityNameFetcher

* Add fixme for relation query processor

* db restore with select all edqs fields

* fixed entity deletion

* fixed FieldUtils with new EntityFields

* dao method renamed

* EDQS: instance groups with same partitions; automatic sync; multiple fixes

* Refactoring for EDQS sync

* EDQS: refactoring

* Fix startup with Kafka

* fixed EntityQueryControllerTest

* fixed EdqsEntityServiceTest

* Separate queue admin for EDQS request template

* Implement new EDQS partitioning strategy

* EDQS: multiple fixes and refactoring

* Add mock EdqsRocksDb beans to tests

* added edqs stats for inmemory/grafana

* fixed filter tests

* Update todos

* Refactoring for QueueConfig

* Improvements and refactoring for EDQS consumers

* implemented TODOs

* test fixes

* Consume state topic up to end offsets

* edqs stats refactoring

* EDQS: cleanup on partitions removal; refactoring

* EDQS: minor refactoring

* EDQS: remove CSV loader

---------

Co-authored-by: Andrii Shvaika <ashvayka@thingsboard.io>
Co-authored-by: dashevchenko <dshevchenko@thingsboard.io>

application/pom.xml

@@ -124,6 +124,10 @@
             <groupId>org.thingsboard.common</groupId>
             <artifactId>edge-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>edqs</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.thingsboard</groupId>
             <artifactId>dao</artifactId>

application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java

@@ -20,6 +20,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
@@ -27,6 +28,7 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.context.request.async.DeferredResult;
+import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.id.UserId;
@@ -38,6 +40,7 @@ import org.thingsboard.server.common.data.query.EntityCountQuery;
 import org.thingsboard.server.common.data.query.EntityData;
 import org.thingsboard.server.common.data.query.EntityDataPageLink;
 import org.thingsboard.server.common.data.query.EntityDataQuery;
+import org.thingsboard.server.common.msg.edqs.EdqsService;
 import org.thingsboard.server.config.annotations.ApiOperation;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.query.EntityQueryService;
@@ -55,6 +58,8 @@ public class EntityQueryController extends BaseController {
 
     @Autowired
     private EntityQueryService entityQueryService;
+    @Autowired
+    private EdqsService edqsService;
 
     private static final int MAX_PAGE_SIZE = 100;
 
@@ -133,4 +138,10 @@ public class EntityQueryController extends BaseController {
         return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope);
     }
 
+    @PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
+    @PostMapping("/edqs/system/request")
+    public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) {
+        edqsService.processSystemRequest(request);
+    }
+
 }

application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java

@@ -82,6 +82,11 @@ public class EdgeEventSourcingListener {
 
     @TransactionalEventListener(fallbackExecution = true)
     public void handleEvent(SaveEntityEvent<?> event) {
+        if (Boolean.FALSE.equals(event.getBroadcastEvent())) {
+            log.trace("Ignoring event {}", event);
+            return;
+        }
+
         try {
             if (!isValidSaveEntityEventForEdgeProcessing(event)) {
                 return;

application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java

@@ -0,0 +1,340 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.edqs;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.protobuf.ByteString;
+import jakarta.annotation.PostConstruct;
+import jakarta.annotation.PreDestroy;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.common.util.ThingsBoardExecutors;
+import org.thingsboard.server.cluster.TbClusterService;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.ObjectType;
+import org.thingsboard.server.common.data.edqs.EdqsEventType;
+import org.thingsboard.server.common.data.edqs.EdqsObject;
+import org.thingsboard.server.common.data.edqs.EdqsSyncRequest;
+import org.thingsboard.server.common.data.edqs.Entity;
+import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
+import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
+import org.thingsboard.server.common.data.edqs.query.EdqsRequest;
+import org.thingsboard.server.common.data.edqs.query.EdqsResponse;
+import org.thingsboard.server.common.data.id.CustomerId;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
+import org.thingsboard.server.common.data.kv.JsonDataEntry;
+import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.common.msg.edqs.EdqsService;
+import org.thingsboard.server.common.msg.queue.ServiceType;
+import org.thingsboard.server.dao.attributes.AttributesService;
+import org.thingsboard.server.edqs.processor.EdqsConverter;
+import org.thingsboard.server.edqs.processor.EdqsProducer;
+import org.thingsboard.server.edqs.util.EdqsPartitionService;
+import org.thingsboard.server.gen.transport.TransportProtos;
+import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.EdqsRequestMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
+import org.thingsboard.server.queue.TbQueueRequestTemplate;
+import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.discovery.HashPartitionService;
+import org.thingsboard.server.queue.edqs.EdqsQueue;
+import org.thingsboard.server.queue.environment.DistributedLock;
+import org.thingsboard.server.queue.environment.DistributedLockService;
+import org.thingsboard.server.queue.provider.EdqsClientQueueFactory;
+import org.thingsboard.server.queue.util.AfterStartUp;
+
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true")
+public class DefaultEdqsService implements EdqsService {
+
+    private final EdqsClientQueueFactory queueFactory;
+    private final EdqsConverter edqsConverter;
+    private final EdqsSyncService edqsSyncService;
+    private final DistributedLockService distributedLockService;
+    private final AttributesService attributesService;
+    private final EdqsPartitionService edqsPartitionService;
+    @Autowired @Lazy
+    private TbClusterService clusterService;
+    @Autowired @Lazy
+    private HashPartitionService hashPartitionService;
+
+    @Value("${queue.edqs.api_enabled:false}")
+    private Boolean apiEnabled;
+
+    private EdqsProducer eventsProducer;
+    private TbQueueRequestTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> requestTemplate;
+    private ExecutorService executor;
+    private DistributedLock<EdqsSyncState> syncLock;
+
+    @PostConstruct
+    private void init() {
+        executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass());
+        eventsProducer = EdqsProducer.builder()
+                .queue(EdqsQueue.EVENTS)
+                .partitionService(edqsPartitionService)
+                .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS))
+                .build();
+        if (apiEnabled) {
+            apiEnabled = null;
+        }
+
+        requestTemplate = queueFactory.createEdqsRequestTemplate();
+        requestTemplate.init();
+        syncLock = distributedLockService.getLock("edqs_sync");
+    }
+
+    @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
+    public void onStartUp() {
+        executor.submit(() -> {
+            try {
+                EdqsSyncState syncState = getSyncState();
+                if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) {
+                    if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) {
+                        processSystemRequest(ToCoreEdqsRequest.builder()
+                                .syncRequest(new EdqsSyncRequest())
+                                .build());
+                    }
+                } else { // only if topic/RocksDB is not empty and sync is finished
+                    if (apiEnabled == null) {
+                        log.info("EDQS is already synced, enabling API");
+                        apiEnabled = true;
+                    } else {
+                        log.info("EDQS is already synced");
+                    }
+                }
+            } catch (Throwable e) {
+                log.error("Failed to start EDQS service", e);
+            }
+        });
+    }
+
+    @Override
+    public void processSystemRequest(ToCoreEdqsRequest request) {
+        log.info("Processing system request {}", request);
+        if (request.getSyncRequest() != null) {
+            saveSyncState(EdqsSyncStatus.REQUESTED);
+        }
+        broadcast(request.toInternalMsg());
+    }
+
+    @Override
+    public void processSystemMsg(ToCoreEdqsMsg msg) {
+        executor.submit(() -> {
+            log.info("Processing system msg {}", msg);
+            try {
+                if (msg.getApiEnabled() != null) {
+                    apiEnabled = msg.getApiEnabled();
+                }
+
+                if (msg.getSyncRequest() != null) {
+                    syncLock.lock();
+                    try {
+                        EdqsSyncState syncState = getSyncState();
+                        if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) {
+                            log.info("EDQS sync is already finished");
+                            return;
+                        }
+
+                        saveSyncState(EdqsSyncStatus.STARTED);
+                        edqsSyncService.sync();
+
+                        saveSyncState(EdqsSyncStatus.FINISHED);
+                        if (apiEnabled == null) {
+                            broadcast(ToCoreEdqsMsg.builder()
+                                    .apiEnabled(Boolean.TRUE)
+                                    .build());
+                        }
+                    } catch (Exception e) {
+                        log.error("Failed to complete sync", e);
+                        saveSyncState(EdqsSyncStatus.FAILED);
+                    } finally {
+                        syncLock.unlock();
+                    }
+                }
+            } catch (Throwable e) {
+                log.error("Failed to process msg {}", msg, e);
+            }
+        });
+    }
+
+    @Override
+    public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {
+        EntityType entityType = entityId.getEntityType();
+        ObjectType objectType = ObjectType.fromEntityType(entityType);
+        if (!isEdqsType(tenantId, objectType)) {
+            log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType);
+            return;
+        }
+        onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity));
+    }
+
+    @Override
+    public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) {
+        processEvent(tenantId, objectType, EdqsEventType.UPDATED, object);
+    }
+
+    @Override
+    public void onDelete(TenantId tenantId, EntityId entityId) {
+        EntityType entityType = entityId.getEntityType();
+        ObjectType objectType = ObjectType.fromEntityType(entityType);
+        if (!isEdqsType(tenantId, objectType)) {
+            log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType);
+            return;
+        }
+        onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE));
+    }
+
+    @Override
+    public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) {
+        processEvent(tenantId, objectType, EdqsEventType.DELETED, object);
+    }
+
+    @Override
+    public ListenableFuture<EdqsResponse> processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) {
+        var requestMsg = newEdqsMsg(tenantId)
+                .setRequestMsg(EdqsRequestMsg.newBuilder()
+                        .setValue(JacksonUtil.toString(request))
+                        .build());
+        if (customerId != null && !customerId.isNullUid()) {
+            requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits());
+            requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits());
+        }
+
+        Integer partition = edqsPartitionService.resolvePartition(tenantId);
+        ListenableFuture<TbProtoQueueMsg<FromEdqsMsg>> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition);
+        return Futures.transform(resultFuture, msg -> {
+            TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg();
+            return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class);
+        }, MoreExecutors.directExecutor());
+    }
+
+    @Override
+    public boolean isApiEnabled() {
+        return Boolean.TRUE.equals(apiEnabled);
+    }
+
+    protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) {
+        executor.submit(() -> {
+            try {
+                String key = object.key();
+                Long version = object.version();
+                EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder()
+                        .setKey(key)
+                        .setObjectType(objectType.name())
+                        .setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object)))
+                        .setEventType(eventType.name());
+                if (version != null) {
+                    eventMsg.setVersion(version);
+                }
+                eventsProducer.send(tenantId, objectType, key, newEdqsMsg(tenantId)
+                        .setEventMsg(eventMsg)
+                        .build());
+            } catch (Throwable e) {
+                log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e);
+            }
+        });
+    }
+
+    private boolean isEdqsType(TenantId tenantId, ObjectType objectType) {
+        if (objectType == null) {
+            return false;
+        }
+        if (!tenantId.isSysTenantId()) {
+            return ObjectType.edqsTypes.contains(objectType);
+        } else {
+            return ObjectType.edqsSystemTypes.contains(objectType);
+        }
+    }
+
+    private void broadcast(ToCoreEdqsMsg msg) {
+        clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder()
+                .setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder()
+                        .setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg))))
+                .build());
+    }
+
+    private static ToEdqsMsg.Builder newEdqsMsg(TenantId tenantId) {
+        return ToEdqsMsg.newBuilder()
+                .setTenantIdMSB(tenantId.getId().getMostSignificantBits())
+                .setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
+                .setTs(System.currentTimeMillis());
+    }
+
+    @PreDestroy
+    private void preDestroy() {
+        executor.shutdown();
+        eventsProducer.stop();
+        requestTemplate.stop();
+    }
+
+    @SneakyThrows
+    private EdqsSyncState getSyncState() {
+        EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS)
+                .flatMap(KvEntry::getJsonValue)
+                .map(value -> JacksonUtil.fromString(value, EdqsSyncState.class))
+                .orElse(null);
+        log.info("getSyncState = {}", state);
+        return state;
+    }
+
+    @SneakyThrows
+    private void saveSyncState(EdqsSyncStatus status) {
+        EdqsSyncState state = new EdqsSyncState(status);
+        log.info("saveSyncState {}", state);
+        attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(
+                new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)),
+                System.currentTimeMillis())).get(30, TimeUnit.SECONDS);
+    }
+
+    @Data
+    @AllArgsConstructor
+    @NoArgsConstructor
+    private static class EdqsSyncState {
+        private EdqsSyncStatus status;
+    }
+
+    private enum EdqsSyncStatus {
+        REQUESTED,
+        STARTED,
+        FINISHED,
+        FAILED
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java

@@ -0,0 +1,539 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.edqs;
+
+import com.fasterxml.jackson.databind.MappingIterator;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
+import org.thingsboard.server.common.data.ApiUsageState;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.Customer;
+import org.thingsboard.server.common.data.Dashboard;
+import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.DeviceProfile;
+import org.thingsboard.server.common.data.DeviceProfileType;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.EntityView;
+import org.thingsboard.server.common.data.ObjectType;
+import org.thingsboard.server.common.data.StringUtils;
+import org.thingsboard.server.common.data.Tenant;
+import org.thingsboard.server.common.data.User;
+import org.thingsboard.server.common.data.asset.Asset;
+import org.thingsboard.server.common.data.asset.AssetProfile;
+import org.thingsboard.server.common.data.converter.Converter;
+import org.thingsboard.server.common.data.converter.ConverterType;
+import org.thingsboard.server.common.data.edge.Edge;
+import org.thingsboard.server.common.data.edqs.AttributeKv;
+import org.thingsboard.server.common.data.edqs.LatestTsKv;
+import org.thingsboard.server.common.data.group.EntityGroup;
+import org.thingsboard.server.common.data.id.ApiUsageStateId;
+import org.thingsboard.server.common.data.id.AssetId;
+import org.thingsboard.server.common.data.id.AssetProfileId;
+import org.thingsboard.server.common.data.id.ConverterId;
+import org.thingsboard.server.common.data.id.CustomerId;
+import org.thingsboard.server.common.data.id.DashboardId;
+import org.thingsboard.server.common.data.id.DeviceId;
+import org.thingsboard.server.common.data.id.DeviceProfileId;
+import org.thingsboard.server.common.data.id.EdgeId;
+import org.thingsboard.server.common.data.id.EntityGroupId;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.EntityIdFactory;
+import org.thingsboard.server.common.data.id.EntityViewId;
+import org.thingsboard.server.common.data.id.IntegrationId;
+import org.thingsboard.server.common.data.id.RoleId;
+import org.thingsboard.server.common.data.id.RuleChainId;
+import org.thingsboard.server.common.data.id.SchedulerEventId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.id.TenantProfileId;
+import org.thingsboard.server.common.data.id.UserId;
+import org.thingsboard.server.common.data.id.WidgetTypeId;
+import org.thingsboard.server.common.data.id.WidgetsBundleId;
+import org.thingsboard.server.common.data.integration.Integration;
+import org.thingsboard.server.common.data.integration.IntegrationType;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
+import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
+import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
+import org.thingsboard.server.common.data.kv.BooleanDataEntry;
+import org.thingsboard.server.common.data.kv.DoubleDataEntry;
+import org.thingsboard.server.common.data.kv.JsonDataEntry;
+import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.common.data.kv.LongDataEntry;
+import org.thingsboard.server.common.data.kv.StringDataEntry;
+import org.thingsboard.server.common.data.relation.EntityRelation;
+import org.thingsboard.server.common.data.relation.RelationTypeGroup;
+import org.thingsboard.server.common.data.role.Role;
+import org.thingsboard.server.common.data.role.RoleType;
+import org.thingsboard.server.common.data.rule.RuleChain;
+import org.thingsboard.server.common.data.scheduler.SchedulerEvent;
+import org.thingsboard.server.common.data.widget.WidgetType;
+import org.thingsboard.server.common.data.widget.WidgetsBundle;
+import org.thingsboard.server.common.msg.edqs.EdqsService;
+import org.thingsboard.server.edqs.processor.EdqsConverter;
+
+import java.io.FileReader;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.function.Consumer;
+
+import static org.thingsboard.common.util.JacksonUtil.toJsonNode;
+
+
+@RequiredArgsConstructor
+@Slf4j
+//@Service
+public class EdqsDataLoader {
+
+    private final EdqsService edqsService;
+    private final EdqsConverter edqsConverter;
+
+    public final static TenantId MAIN = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60"));
+
+    private final String folder = "/home/viacheslav/Downloads/schwarz";
+
+    private ExecutorService executor = Executors.newFixedThreadPool(5, ThingsBoardThreadFactory.forName("edqs-publisher"));
+
+//    @AfterStartUp(order = 100)
+    public void load() throws Exception {
+        loadCustomers();
+        loadDeviceProfile();
+        loadDevices();
+        loadAssets();
+        loadEdges();
+        loadEntityViews();
+        loadTenants();
+        loadUsers();
+        loadDashboards();
+        loadRuleChains();
+        loadWidgetType();
+        loadWidgetBundle();
+        loadConverters();
+        loadIntegrations();
+        loadSchedulerEvents();
+        loadRoles();
+        loadApiUsageStates();
+        loadAssetProfile();
+        loadEntityGroups();
+        loadRelations();
+
+        loadAttributes();
+        loadTs();
+    }
+
+    private void loadCustomers() throws Exception {
+        load("customer.csv", (values) -> {
+            Customer customer = new Customer();
+            customer.setTitle(values.get("title"));
+            customer.setId(new CustomerId(UUID.fromString(values.get("id"))));
+            customer.setCreatedTime(Long.parseLong(values.get("created_time")));
+            customer.setTenantId(tenantId(values.get("tenant_id")));
+            var parentCustomerId = values.get("parent_customer_id");
+            if (StringUtils.isNotEmpty(parentCustomerId)) {
+                customer.setParentCustomerId(new CustomerId(UUID.fromString(parentCustomerId)));
+            }
+            edqsService.onUpdate(customer.getTenantId(), customer.getId(), customer);
+        });
+    }
+
+    private void loadDevices() throws Exception {
+        load("device.csv", (values) -> {
+            Device device = new Device();
+            device.setType(values.get("type"));
+            device.setName(values.get("name"));
+            device.setLabel(values.get("label"));
+            device.setId(new DeviceId(uuid(values.get("id"))));
+            device.setCreatedTime(parseLong(values.get("created_time")));
+            device.setCustomerId(customerId(values.get("customer_id")));
+            device.setTenantId(tenantId(values.get("tenant_id")));
+            device.setDeviceProfileId(new DeviceProfileId(uuid(values.get("device_profile_id"))));
+            device.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(device.getTenantId(), device.getId(), device);
+        });
+    }
+
+    private void loadAssets() throws Exception {
+        load("asset.csv", (values) -> {
+            Asset asset = new Asset();
+            asset.setType(values.get("type"));
+            asset.setName(values.get("name"));
+            asset.setLabel(values.get("label"));
+            asset.setId(new AssetId(uuid(values.get("id"))));
+            asset.setCreatedTime(parseLong(values.get("created_time")));
+            asset.setCustomerId(customerId(values.get("customer_id")));
+            asset.setTenantId(tenantId(values.get("tenant_id")));
+            asset.setAssetProfileId(new AssetProfileId(uuid(values.get("asset_profile_id"))));
+            asset.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(asset.getTenantId(), asset.getId(), asset);
+        });
+    }
+
+    private void loadEdges() throws Exception {
+        load("edge.csv", (values) -> {
+            Edge edge = new Edge();
+            edge.setId(new EdgeId(uuid(values.get("id"))));
+            edge.setCreatedTime(parseLong(values.get("created_time")));
+            edge.setType(values.get("type"));
+            edge.setName(values.get("name"));
+            edge.setLabel(values.get("label"));
+            edge.setCustomerId(customerId(values.get("customer_id")));
+            edge.setTenantId(tenantId(values.get("tenant_id")));
+            edge.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(edge.getTenantId(), edge.getId(), edge);
+        });
+    }
+
+    private void loadEntityViews() throws Exception {
+        load("entity_view.csv", (values) -> {
+            EntityView entityView = new EntityView();
+            entityView.setId(new EntityViewId(uuid(values.get("id"))));
+            entityView.setCreatedTime(parseLong(values.get("created_time")));
+            entityView.setType(values.get("type"));
+            entityView.setName(values.get("name"));
+            entityView.setCustomerId(customerId(values.get("customer_id")));
+            entityView.setTenantId(tenantId(values.get("tenant_id")));
+            entityView.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(entityView.getTenantId(), entityView.getId(), entityView);
+        });
+    }
+
+    private void loadTenants() throws Exception {
+        load("tenant.csv", (values) -> {
+            Tenant tenant = new Tenant();
+            tenant.setId(new TenantId(uuid(values.get("id"))));
+            tenant.setCreatedTime(parseLong(values.get("created_time")));
+            tenant.setEmail(values.get("email"));
+            tenant.setTitle(values.get("title"));
+            tenant.setCountry(values.get("country"));
+            tenant.setState(values.get("state"));
+            tenant.setCity(values.get("city"));
+            tenant.setAddress(values.get("address"));
+            tenant.setAddress2(values.get("address2"));
+            tenant.setZip(values.get("zip"));
+            tenant.setPhone(values.get("phone"));
+            tenant.setRegion(values.get("region"));
+            tenant.setTenantProfileId(new TenantProfileId(uuid(values.get("tenant_profile_id"))));
+            tenant.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+            edqsService.onUpdate(MAIN, tenant.getId(), tenant);
+        });
+    }
+
+    private void loadUsers() throws Exception {
+        load("user.csv", (values) -> {
+            User user = new User();
+            user.setId(new UserId(uuid(values.get("id"))));
+            user.setCreatedTime(parseLong(values.get("created_time")));
+            user.setTenantId(tenantId(values.get("tenant_id")));
+            user.setFirstName(values.get("first_name"));
+            user.setLastName(values.get("last_name"));
+            user.setEmail(values.get("email"));
+            user.setPhone(values.get("phone"));
+            user.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(user.getTenantId(), user.getId(), user);
+        });
+    }
+
+    private void loadDashboards() throws Exception {
+        load("dashboard.csv", (values) -> {
+            Dashboard dashboard = new Dashboard();
+            dashboard.setId(new DashboardId(uuid(values.get("id"))));
+            dashboard.setCreatedTime(parseLong(values.get("created_time")));
+            dashboard.setTenantId(tenantId(values.get("tenant_id")));
+            dashboard.setTitle(values.get("title"));
+
+            edqsService.onUpdate(dashboard.getTenantId(), dashboard.getId(), dashboard);
+        });
+    }
+
+    private void loadEntityGroups() throws Exception {
+        load("entity_group.csv", (values) -> {
+            EntityGroup entityGroup = new EntityGroup();
+            entityGroup.setId(new EntityGroupId(uuid(values.get("id"))));
+            entityGroup.setCreatedTime(parseLong(values.get("created_time")));
+            entityGroup.setName(values.get("name"));
+            entityGroup.setOwnerId(entityId(values.get("owner_type"), values.get("owner_id")));
+            entityGroup.setType(EntityType.valueOf(values.get("type")));
+            edqsService.onUpdate(MAIN, entityGroup.getId(), entityGroup);
+        });
+    }
+
+    private void loadRelations() throws Exception {
+        load("relation.csv", (values) -> {
+            EntityRelation entityRelation = new EntityRelation();
+            entityRelation.setFrom(entityId(values.get("from_type"), values.get("from_id")));
+            entityRelation.setTo(entityId(values.get("to_type"), values.get("to_id")));
+            entityRelation.setTypeGroup(RelationTypeGroup.valueOf(values.get("relation_type_group")));
+            entityRelation.setType(values.get("relation_type"));
+            edqsService.onUpdate(MAIN, ObjectType.RELATION, entityRelation);
+        });
+    }
+
+    private void loadRuleChains() throws Exception {
+        load("rule_chain.csv", (values) -> {
+            RuleChain ruleChain = new RuleChain();
+            ruleChain.setId(new RuleChainId(uuid(values.get("id"))));
+            ruleChain.setCreatedTime(parseLong(values.get("created_time")));
+            ruleChain.setName(values.get("name"));
+            ruleChain.setTenantId(tenantId(values.get("tenant_id")));
+            ruleChain.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(ruleChain.getTenantId(), ruleChain.getId(), ruleChain);
+        });
+    }
+
+    private void loadWidgetType() throws Exception {
+        load("widget_type.csv", (values) -> {
+            WidgetType widgetType = new WidgetType();
+            widgetType.setId(new WidgetTypeId(uuid(values.get("id"))));
+            widgetType.setCreatedTime(parseLong(values.get("created_time")));
+            widgetType.setName(values.get("name"));
+            widgetType.setTenantId(tenantId(values.get("tenant_id")));
+
+            edqsService.onUpdate(widgetType.getTenantId(), widgetType.getId(), widgetType);
+        });
+    }
+
+    private void loadWidgetBundle() throws Exception {
+        load("widgets_bundle.csv", (values) -> {
+            WidgetsBundle widgetsBundle = new WidgetsBundle();
+            widgetsBundle.setId(new WidgetsBundleId(uuid(values.get("id"))));
+            widgetsBundle.setCreatedTime(parseLong(values.get("created_time")));
+            widgetsBundle.setTitle(values.get("title"));
+            widgetsBundle.setTenantId(tenantId(values.get("tenant_id")));
+
+            edqsService.onUpdate(widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle);
+        });
+    }
+
+    private void loadConverters() throws Exception {
+        load("converter.csv", (values) -> {
+            Converter converter = new Converter();
+            converter.setId(new ConverterId(uuid(values.get("id"))));
+            converter.setCreatedTime(parseLong(values.get("created_time")));
+            converter.setName(values.get("name"));
+            converter.setType(ConverterType.valueOf(values.get("type")));
+            converter.setTenantId(tenantId(values.get("tenant_id")));
+            converter.setEdgeTemplate(parseBoolean(values.get("is_edge_template")));
+            converter.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(converter.getTenantId(), converter.getId(), converter);
+        });
+    }
+
+    private void loadIntegrations() throws Exception {
+        load("integration.csv", (values) -> {
+            Integration integration = new Integration();
+            integration.setId(new IntegrationId(uuid(values.get("id"))));
+            integration.setCreatedTime(parseLong(values.get("created_time")));
+            integration.setName(values.get("name"));
+            integration.setType(IntegrationType.valueOf(values.get("type")));
+            integration.setTenantId(tenantId(values.get("tenant_id")));
+            integration.setEdgeTemplate(parseBoolean(values.get("is_edge_template")));
+            integration.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(integration.getTenantId(), integration.getId(), integration);
+        });
+    }
+
+    private void loadSchedulerEvents() throws Exception {
+        load("scheduler_event.csv", (values) -> {
+            SchedulerEvent schedulerEvent = new SchedulerEvent();
+            schedulerEvent.setId(new SchedulerEventId(uuid(values.get("id"))));
+            schedulerEvent.setCreatedTime(parseLong(values.get("created_time")));
+            schedulerEvent.setName(values.get("name"));
+            schedulerEvent.setType(values.get("type"));
+            schedulerEvent.setTenantId(tenantId(values.get("tenant_id")));
+            schedulerEvent.setConfiguration(toJsonNode(values.get("configuration")));
+            schedulerEvent.setSchedule(toJsonNode(values.get("schedule")));
+            schedulerEvent.setOriginatorId(entityId(values.get("originator_type"), values.get("originator_id")));
+            schedulerEvent.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(schedulerEvent.getTenantId(), schedulerEvent.getId(), schedulerEvent);
+        });
+    }
+
+    private void loadRoles() throws Exception {
+        load("role.csv", (values) -> {
+            Role role = new Role();
+            role.setId(new RoleId(uuid(values.get("id"))));
+            role.setCreatedTime(parseLong(values.get("created_time")));
+            role.setName(values.get("name"));
+            role.setType(RoleType.valueOf(values.get("type")));
+            role.setTenantId(tenantId(values.get("tenant_id")));
+            role.setAdditionalInfo(toJsonNode(values.get("additional_info")));
+
+            edqsService.onUpdate(role.getTenantId(), role.getId(), role);
+        });
+    }
+
+    private void loadApiUsageStates() throws Exception {
+        load("api_usage_state.csv", (values) -> {
+            ApiUsageState apiUsageState = new ApiUsageState();
+            apiUsageState.setId(new ApiUsageStateId(uuid(values.get("id"))));
+            apiUsageState.setCreatedTime(parseLong(values.get("created_time")));
+            apiUsageState.setEntityId(entityId(values.get("entity_type"), values.get("entity_id")));
+            apiUsageState.setTenantId(tenantId(values.get("tenant_id")));
+
+            edqsService.onUpdate(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState);
+        });
+    }
+
+    private void loadDeviceProfile() throws Exception {
+        load("device_profile.csv", (values) -> {
+            DeviceProfile deviceProfile = new DeviceProfile();
+            deviceProfile.setId(new DeviceProfileId(uuid(values.get("id"))));
+            deviceProfile.setCreatedTime(parseLong(values.get("created_time")));
+            deviceProfile.setName(values.get("name"));
+            deviceProfile.setType(DeviceProfileType.valueOf(values.get("type")));
+            deviceProfile.setTenantId(tenantId(values.get("tenant_id")));
+
+            edqsService.onUpdate(deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile);
+        });
+    }
+
+    private void loadAssetProfile() throws Exception {
+        load("asset_profile.csv", (values) -> {
+            AssetProfile assetProfile = new AssetProfile();
+            assetProfile.setId(new AssetProfileId(uuid(values.get("id"))));
+            assetProfile.setCreatedTime(parseLong(values.get("created_time")));
+            assetProfile.setName(values.get("name"));
+            assetProfile.setTenantId(tenantId(values.get("tenant_id")));
+
+            edqsService.onUpdate(assetProfile.getTenantId(), assetProfile.getId(), assetProfile);
+        });
+    }
+
+    private void loadAttributes() throws Exception {
+        load("attribute.csv", (values) -> {
+            EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("entity_type"), values.get("entity_id"));
+            long ts = parseLong(values.get("last_update_ts"));
+            AttributeScope scope = AttributeScope.valueOf(values.get("attribute_type"));
+            String key = values.get("attribute_key");
+            KvEntry kvEntry;
+            if (StringUtils.isNotEmpty(values.get("bool_v"))) {
+                kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v")));
+            } else if (StringUtils.isNotEmpty(values.get("str_v"))) {
+                kvEntry = new StringDataEntry(key, values.get("str_v"));
+            } else if (StringUtils.isNotEmpty(values.get("long_v"))) {
+                kvEntry = new LongDataEntry(key, parseLong(values.get("long_v")));
+            } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) {
+                kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v")));
+            } else if (StringUtils.isNotEmpty(values.get("json_v"))) {
+                kvEntry = new JsonDataEntry(key, values.get("json_v"));
+            } else {
+                kvEntry = new StringDataEntry(key, "");
+            }
+            AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(ts, kvEntry);
+            AttributeKv attributeKv = new AttributeKv(entityId, scope, attributeKvEntry, 0);
+            edqsService.onUpdate(MAIN, ObjectType.ATTRIBUTE_KV, attributeKv);
+        });
+    }
+
+    private void loadTs() throws Exception {
+        load("ts_kv.csv", (values) -> {
+            var entityTypeStr = values.get("find_entity_type");
+            if (StringUtils.isEmpty(entityTypeStr)) {
+                return;
+            }
+            EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("find_entity_type"), values.get("entity_id"));
+            long ts = parseLong(values.get("ts"));
+            String key = values.get("key");
+            KvEntry kvEntry;
+            if (StringUtils.isNotEmpty(values.get("bool_v"))) {
+                kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v")));
+            } else if (StringUtils.isNotEmpty(values.get("str_v"))) {
+                kvEntry = new StringDataEntry(key, values.get("str_v"));
+            } else if (StringUtils.isNotEmpty(values.get("long_v"))) {
+                kvEntry = new LongDataEntry(key, parseLong(values.get("long_v")));
+            } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) {
+                kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v")));
+            } else if (StringUtils.isNotEmpty(values.get("json_v"))) {
+                kvEntry = new JsonDataEntry(key, values.get("json_v"));
+            } else {
+                kvEntry = new StringDataEntry(key, "");
+            }
+            BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(ts, kvEntry);
+            edqsService.onUpdate(MAIN, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, 0L));
+        });
+    }
+
+    private void load(String file, Consumer<Map<String, String>> function) throws Exception {
+        Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("loader-" + file)).submit(() -> {
+            try {
+                long ts = System.currentTimeMillis();
+                CsvSchema schema = CsvSchema.emptySchema().withHeader().withColumnSeparator('|');
+                CsvMapper mapper = new CsvMapper();
+                MappingIterator<Map<String, String>> it = mapper
+                        .readerFor(Map.class)
+                        .with(schema)
+                        .readValues(new FileReader(folder + "/" + file));
+
+                int success = 0;
+                int failure = 0;
+                while (it.hasNextValue()) {
+                    Map<String, String> row = it.nextValue();
+                    try {
+                        function.accept(row);
+                        success++;
+                        if (success % 1000 == 0) {
+                            log.info("Loaded [{}] from [{}]", success, file);
+                        }
+                    } catch (Exception e) {
+                        log.error("Failed to parse str: [{}]", row, e);
+                        failure++;
+                    }
+                }
+                log.info("Loaded [{}] from [{}] in {}ms. Failures {}", success, file, (System.currentTimeMillis() - ts), failure);
+            } catch (Throwable t) {
+                log.error("Failed to load data from [{}]", file, t);
+            }
+        });
+    }
+
+    private static TenantId tenantId(String id) {
+        return TenantId.fromUUID(UUID.fromString(id));
+    }
+
+    private static CustomerId customerId(String id) {
+        var c = new CustomerId(UUID.fromString(id));
+        return c.isNullUid() ? null : c;
+    }
+
+    private static EntityId entityId(String type, String id) {
+        return EntityIdFactory.getByTypeAndId(type, id);
+    }
+
+    private static UUID uuid(String id) {
+        return UUID.fromString(id);
+    }
+
+    private static long parseLong(String time) {
+        return Long.parseLong(time);
+    }
+
+    private static boolean parseBoolean(String value) {
+        return Boolean.parseBoolean(value);
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java

@@ -0,0 +1,61 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.edqs;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.event.TransactionalEventListener;
+import org.thingsboard.server.common.data.ObjectType;
+import org.thingsboard.server.common.data.audit.ActionType;
+import org.thingsboard.server.common.msg.edqs.EdqsService;
+import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
+import org.thingsboard.server.dao.eventsourcing.RelationActionEvent;
+import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
+
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true")
+public class EdqsListener {
+
+    private final EdqsService edqsService;
+
+    @TransactionalEventListener(fallbackExecution = true)
+    public void onUpdate(SaveEntityEvent<?> event) {
+        if (event.getEntityId() == null || event.getEntity() == null) {
+            return;
+        }
+        edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity());
+    }
+
+    @TransactionalEventListener(fallbackExecution = true)
+    public void onDelete(DeleteEntityEvent<?> event) {
+        if (event.getEntityId() == null) {
+            return;
+        }
+        edqsService.onDelete(event.getTenantId(), event.getEntityId());
+    }
+
+    @TransactionalEventListener(fallbackExecution = true)
+    public void handleEvent(RelationActionEvent relationEvent) {
+        if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) {
+            edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation());
+        } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) {
+            edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation());
+        }
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java

@@ -0,0 +1,275 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.edqs;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.ObjectType;
+import org.thingsboard.server.common.data.edqs.AttributeKv;
+import org.thingsboard.server.common.data.edqs.EdqsEventType;
+import org.thingsboard.server.common.data.edqs.EdqsObject;
+import org.thingsboard.server.common.data.edqs.Entity;
+import org.thingsboard.server.common.data.edqs.LatestTsKv;
+import org.thingsboard.server.common.data.edqs.fields.EntityFields;
+import org.thingsboard.server.common.data.edqs.fields.TenantFields;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.EntityIdFactory;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.page.PageDataIterable;
+import org.thingsboard.server.common.data.page.SortOrder;
+import org.thingsboard.server.common.data.relation.EntityRelation;
+import org.thingsboard.server.common.data.relation.RelationTypeGroup;
+import org.thingsboard.server.dao.Dao;
+import org.thingsboard.server.dao.attributes.AttributesDao;
+import org.thingsboard.server.dao.dictionary.KeyDictionaryDao;
+import org.thingsboard.server.dao.entity.EntityDaoRegistry;
+import org.thingsboard.server.dao.group.EntityGroupDao;
+import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
+import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry;
+import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
+import org.thingsboard.server.dao.relation.RelationDao;
+import org.thingsboard.server.dao.tenant.TenantDao;
+import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao;
+
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.thingsboard.server.common.data.ObjectType.API_USAGE_STATE;
+import static org.thingsboard.server.common.data.ObjectType.ASSET;
+import static org.thingsboard.server.common.data.ObjectType.ASSET_PROFILE;
+import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV;
+import static org.thingsboard.server.common.data.ObjectType.BLOB_ENTITY;
+import static org.thingsboard.server.common.data.ObjectType.CONVERTER;
+import static org.thingsboard.server.common.data.ObjectType.CUSTOMER;
+import static org.thingsboard.server.common.data.ObjectType.DASHBOARD;
+import static org.thingsboard.server.common.data.ObjectType.DEVICE;
+import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE;
+import static org.thingsboard.server.common.data.ObjectType.EDGE;
+import static org.thingsboard.server.common.data.ObjectType.ENTITY_GROUP;
+import static org.thingsboard.server.common.data.ObjectType.ENTITY_VIEW;
+import static org.thingsboard.server.common.data.ObjectType.INTEGRATION;
+import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV;
+import static org.thingsboard.server.common.data.ObjectType.QUEUE_STATS;
+import static org.thingsboard.server.common.data.ObjectType.RELATION;
+import static org.thingsboard.server.common.data.ObjectType.ROLE;
+import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN;
+import static org.thingsboard.server.common.data.ObjectType.SCHEDULER_EVENT;
+import static org.thingsboard.server.common.data.ObjectType.TENANT;
+import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE;
+import static org.thingsboard.server.common.data.ObjectType.USER;
+import static org.thingsboard.server.common.data.ObjectType.WIDGETS_BUNDLE;
+import static org.thingsboard.server.common.data.ObjectType.WIDGET_TYPE;
+
+@Slf4j
+public abstract class EdqsSyncService {
+
+    @Autowired
+    private EntityDaoRegistry entityDaoRegistry;
+    @Autowired
+    private TenantDao tenantDao;
+    @Autowired
+    private AttributesDao attributesDao;
+    @Autowired
+    private KeyDictionaryDao keyDictionaryDao;
+    @Autowired
+    private RelationDao relationDao;
+    @Autowired
+    private EntityGroupDao entityGroupDao;
+    @Autowired
+    private TimeseriesLatestDao timeseriesLatestDao;
+    @Autowired
+    @Lazy
+    private DefaultEdqsService edqsService;
+
+    private final ConcurrentHashMap<UUID, EntityIdInfo> entityInfoMap = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<Integer, String> keys = new ConcurrentHashMap<>();
+
+    private final Map<ObjectType, AtomicInteger> counters = new ConcurrentHashMap<>();
+
+    public static final Set<ObjectType> edqsTenantTypes = EnumSet.of(
+            TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD,
+            RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE,
+            BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS
+    );
+
+    public abstract boolean isSyncNeeded();
+
+    public void sync() {
+        log.info("Synchronizing data to EDQS");
+        long startTs = System.currentTimeMillis();
+        counters.clear();
+
+        syncTenants();
+        syncTenantEntities();
+        syncEntityGroups();
+        syncRelations();
+        loadKeyDictionary();
+        syncAttributes();
+        syncLatestTimeseries();
+
+        counters.clear();
+        log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs));
+    }
+
+    private void process(TenantId tenantId, ObjectType type, EdqsObject object) {
+        AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger());
+        if (counter.incrementAndGet() % 10000 == 0) {
+            log.info("Processed {} {} objects", counter.get(), type);
+        }
+        edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object);
+    }
+
+    private void syncTenants() {
+        log.info("Synchronizing tenants to EDQS");
+        long ts = System.currentTimeMillis();
+        var tenants = new PageDataIterable<>(tenantDao::findAllFields, 10000);
+        for (EntityFields entityFields : tenants) {
+            TenantId tenantId = TenantId.fromUUID(entityFields.getId());
+            entityInfoMap.put(entityFields.getId(), new EntityIdInfo(EntityType.TENANT, tenantId));
+            process(tenantId, TENANT, new Entity(EntityType.TENANT, entityFields));
+        }
+        process(TenantId.SYS_TENANT_ID, TENANT, new Entity(EntityType.TENANT, new TenantFields(TenantId.SYS_TENANT_ID.getId(), Long.MAX_VALUE)));
+        log.info("Finished synchronizing tenants to EDQS in {} ms", (System.currentTimeMillis() - ts));
+    }
+
+    private void syncTenantEntities() {
+        for (ObjectType type : edqsTenantTypes) {
+            log.info("Synchronizing tenant {} entities to EDQS", type);
+            long ts = System.currentTimeMillis();
+            EntityType entityType = type.toEntityType();
+            Dao<?> dao = entityDaoRegistry.getDao(entityType);
+            var entities = new PageDataIterable<>(dao::findAllFields, 10000);
+            for (EntityFields entityFields : entities) {
+                TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId());
+                entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId));
+                process(tenantId, type, new Entity(type.toEntityType(), entityFields));
+            }
+            log.info("Finished synchronizing tenant {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts));
+        }
+    }
+
+    private void syncEntityGroups() {
+        log.info("Synchronizing entity groups to EDQS");
+        long ts = System.currentTimeMillis();
+        var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 10000);
+        for (EntityFields groupFields : entityGroups) {
+            EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId());
+            if (entityIdInfo != null) {
+                entityInfoMap.put(groupFields.getId(), new EntityIdInfo(EntityType.ENTITY_GROUP, entityIdInfo.tenantId()));
+                process(entityIdInfo.tenantId(), ENTITY_GROUP, new Entity(EntityType.ENTITY_GROUP, groupFields));
+            } else {
+                log.info("Entity group owner not found: " + groupFields.getOwnerId());
+            }
+        }
+        log.info("Finished synchronizing entity groups to EDQS in {} ms", (System.currentTimeMillis() - ts));
+    }
+
+    private void syncRelations() {
+        log.info("Synchronizing relations to EDQS");
+        long ts = System.currentTimeMillis();
+        var relations = new PageDataIterable<>(relationDao::findAll, 10000);
+        for (EntityRelation relation : relations) {
+            if (relation.getTypeGroup() == RelationTypeGroup.COMMON || relation.getTypeGroup() == RelationTypeGroup.FROM_ENTITY_GROUP) {
+                EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFrom().getId());
+                if (entityIdInfo != null) {
+                    process(entityIdInfo.tenantId(), RELATION, relation);
+                } else {
+                    log.info("Relation from entity not found: " + relation.getFrom());
+                }
+            }
+        }
+        log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts));
+    }
+
+    private void loadKeyDictionary() {
+        log.info("Loading key dictionary");
+        long ts = System.currentTimeMillis();
+        var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000);
+        for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) {
+            keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey());
+        }
+        log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts));
+    }
+
+    private void syncAttributes() {
+        log.info("Synchronizing attributes to EDQS");
+        long ts = System.currentTimeMillis();
+        var attributes = new PageDataIterable<>(attributesDao::findAll, 10000);
+        for (AttributeKvEntity attribute : attributes) {
+            attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey()));
+            UUID entityId = attribute.getId().getEntityId();
+            EntityIdInfo entityIdInfo = entityInfoMap.get(entityId);
+            if (entityIdInfo == null) {
+                log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId);
+                continue;
+            }
+            AttributeKv attributeKv = new AttributeKv(
+                    EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId),
+                    AttributeScope.valueOf(attribute.getId().getAttributeType()),
+                    attribute.toData(),
+                    attribute.getVersion());
+            process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv);
+        }
+        log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts));
+    }
+
+    private void syncLatestTimeseries() {
+        log.info("Synchronizing latest timeseries to EDQS");
+        long ts = System.currentTimeMillis();
+        var tsKvLatestEntities = new PageDataIterable<>(pageLink -> timeseriesLatestDao.findAllLatest(pageLink), 10000);
+        for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) {
+            try {
+                String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey());
+                if (strKey == null) {
+                    log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey());
+                    continue;
+                }
+                tsKvLatestEntity.setStrKey(strKey);
+                UUID entityUuid = tsKvLatestEntity.getEntityId();
+                EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid);
+                if (entityIdInfo != null) {
+                    EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid);
+                    LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion());
+                    process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv);
+                }
+            } catch (Exception e) {
+                log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e);
+            }
+        }
+        log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts));
+    }
+
+    private String getStrKeyOrFetchFromDb(int key) {
+        String strKey = keys.get(key);
+        if (strKey != null) {
+            return strKey;
+        } else {
+            strKey = keyDictionaryDao.getKey(key);
+            keys.putIfAbsent(key, strKey);
+        }
+        return strKey;
+    }
+
+    public record EntityIdInfo(EntityType entityType, TenantId tenantId) {}
+
+}

application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java

@@ -0,0 +1,47 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.edqs;
+
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.queue.edqs.EdqsQueue;
+import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
+import org.thingsboard.server.queue.kafka.TbKafkaSettings;
+
+import java.util.Collections;
+
+@Service
+@RequiredArgsConstructor
+@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'")
+public class KafkaEdqsSyncService extends EdqsSyncService {
+
+    private final TbKafkaSettings kafkaSettings;
+    private TbKafkaAdmin kafkaAdmin;
+
+    @PostConstruct
+    private void init() {
+        kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap());
+    }
+
+    @Override
+    public boolean isSyncNeeded() {
+        return kafkaAdmin.isTopicEmpty(EdqsQueue.STATE.getTopic());
+    }
+
+
+}

application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java

@@ -0,0 +1,35 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.edqs;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.edqs.util.EdqsRocksDb;
+
+@Service
+@RequiredArgsConstructor
+@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'")
+public class LocalEdqsSyncService extends EdqsSyncService {
+
+    private final EdqsRocksDb db;
+
+    @Override
+    public boolean isSyncNeeded() {
+        return db.isNew();
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java

@@ -72,6 +72,11 @@ public class EntityStateSourcingListener {
 
     @TransactionalEventListener(fallbackExecution = true)
     public void handleEvent(SaveEntityEvent<?> event) {
+        if (Boolean.FALSE.equals(event.getBroadcastEvent())) {
+            log.trace("Ignoring event {}", event);
+            return;
+        }
+
         TenantId tenantId = event.getTenantId();
         EntityId entityId = event.getEntityId();
         if (entityId == null) {

application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import jakarta.annotation.PostConstruct;
 import jakarta.annotation.PreDestroy;
-import lombok.Data;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -35,6 +34,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.server.actors.ActorSystemContext;
 import org.thingsboard.server.common.data.JavaSerDesUtil;
 import org.thingsboard.server.common.data.alarm.AlarmInfo;
+import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
 import org.thingsboard.server.common.data.event.ErrorEvent;
 import org.thingsboard.server.common.data.event.Event;
 import org.thingsboard.server.common.data.event.LifecycleEvent;
@@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.queue.QueueConfig;
 import org.thingsboard.server.common.data.rpc.RpcError;
 import org.thingsboard.server.common.msg.MsgType;
 import org.thingsboard.server.common.msg.TbActorMsg;
+import org.thingsboard.server.common.msg.edqs.EdqsService;
 import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
 import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.common.msg.queue.TbCallback;
@@ -78,6 +79,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
 import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
 import org.thingsboard.server.queue.TbQueueConsumer;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager;
 import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
 import org.thingsboard.server.queue.discovery.PartitionService;
 import org.thingsboard.server.queue.discovery.QueueKey;
@@ -89,7 +91,6 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService;
 import org.thingsboard.server.service.ota.OtaPackageStateService;
 import org.thingsboard.server.service.profile.TbAssetProfileCache;
 import org.thingsboard.server.service.profile.TbDeviceProfileCache;
-import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager;
 import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
 import org.thingsboard.server.service.queue.processing.IdMsgPair;
 import org.thingsboard.server.service.resource.TbImageService;
@@ -147,9 +148,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
     private final TbCoreQueueFactory queueFactory;
     private final TbImageService imageService;
     private final RuleEngineCallService ruleEngineCallService;
+    private final EdqsService edqsService;
     private final TbCoreConsumerStats stats;
 
-    private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig> mainConsumer;
+    private MainQueueConsumerManager<TbProtoQueueMsg<ToCoreMsg>, QueueConfig> mainConsumer;
     private QueueConsumerManager<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
     private QueueConsumerManager<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
 
@@ -175,7 +177,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
                                         NotificationSchedulerService notificationSchedulerService,
                                         NotificationRuleProcessor notificationRuleProcessor,
                                         TbImageService imageService,
-                                        RuleEngineCallService ruleEngineCallService) {
+                                        RuleEngineCallService ruleEngineCallService,
+                                        EdqsService edqsService) {
         super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService,
                 eventPublisher, jwtSettingsService);
         this.stateService = stateService;
@@ -191,6 +194,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
         this.imageService = imageService;
         this.ruleEngineCallService = ruleEngineCallService;
         this.queueFactory = tbCoreQueueFactory;
+        this.edqsService = edqsService;
     }
 
     @PostConstruct
@@ -198,9 +202,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
         super.init("tb-core");
         this.deviceActivityEventsExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-device-activity-events-executor")));
 
-        this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, CoreQueueConfig>builder()
+        this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToCoreMsg>, QueueConfig>builder()
                 .queueKey(new QueueKey(ServiceType.TB_CORE))
-                .config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval))
+                .config(QueueConfig.of(consumerPerPartition, pollInterval))
                 .msgPackProcessor(this::processMsgs)
                 .consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer())
                 .consumerExecutor(consumersExecutor)
@@ -251,7 +255,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
                 .collect(Collectors.toSet()));
     }
 
-    private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, CoreQueueConfig config) throws Exception {
+    private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, QueueConfig config) throws Exception {
         List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
         ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
                 Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
@@ -396,6 +400,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
             callback.onSuccess();
         } else if (toCoreNotification.hasResourceCacheInvalidateMsg()) {
             forwardToResourceService(toCoreNotification.getResourceCacheInvalidateMsg(), callback);
+        } else if (toCoreNotification.hasToEdqsCoreServiceMsg()) {
+            edqsService.processSystemMsg(JacksonUtil.fromBytes(toCoreNotification.getToEdqsCoreServiceMsg().getValue().toByteArray(), ToCoreEdqsMsg.class));
+            callback.onSuccess();
         }
         if (statsEnabled) {
             stats.log(toCoreNotification);
@@ -730,10 +737,4 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
         firmwareStatesConsumer.stop();
     }
 
-    @Data(staticConstructor = "of")
-    public static class CoreQueueConfig implements QueueConfig {
-        private final boolean consumerPerPartition;
-        private final int pollInterval;
-    }
-
 }

application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java

@@ -52,7 +52,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.edge.EdgeContextComponent;
-import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager;
+import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager;
 import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
 import org.thingsboard.server.service.queue.processing.IdMsgPair;
 
@@ -87,7 +87,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
     private final EdgeContextComponent edgeCtx;
     private final EdgeConsumerStats stats;
 
-    private MainQueueConsumerManager<TbProtoQueueMsg<ToEdgeMsg>, EdgeQueueConfig> mainConsumer;
+    private MainQueueConsumerManager<TbProtoQueueMsg<ToEdgeMsg>, QueueConfig> mainConsumer;
 
     public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext,
                                         StatsFactory statsFactory, EdgeContextComponent edgeCtx) {
@@ -102,9 +102,9 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
     public void init() {
         super.init("tb-edge");
 
-        this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToEdgeMsg>, EdgeQueueConfig>builder()
+        this.mainConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<ToEdgeMsg>, QueueConfig>builder()
                 .queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME))
-                .config(EdgeQueueConfig.of(consumerPerPartition, pollInterval))
+                .config(QueueConfig.of(consumerPerPartition, pollInterval))
                 .msgPackProcessor(this::processMsgs)
                 .consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer())
                 .consumerExecutor(consumersExecutor)
@@ -130,7 +130,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
         mainConsumer.update(partitions);
     }
 
-    private void processMsgs(List<TbProtoQueueMsg<ToEdgeMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeMsg>> consumer, EdgeQueueConfig edgeQueueConfig) throws InterruptedException {
+    private void processMsgs(List<TbProtoQueueMsg<ToEdgeMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeMsg>> consumer, QueueConfig edgeQueueConfig) throws InterruptedException {
         List<IdMsgPair<ToEdgeMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
         ConcurrentMap<UUID, TbProtoQueueMsg<ToEdgeMsg>> pendingMap = orderedMsgList.stream().collect(
                 Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
@@ -323,10 +323,4 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
         mainConsumer.awaitStop();
     }
 
-    @Data(staticConstructor = "of")
-    public static class EdgeQueueConfig implements QueueConfig {
-        private final boolean consumerPerPartition;
-        private final int pollInterval;
-    }
-
 }

application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@@ -104,7 +104,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
 
     @Override
     protected void onTbApplicationEvent(PartitionChangeEvent event) {
-        event.getPartitionsMap().forEach((queueKey, partitions) -> {
+        event.getNewPartitions().forEach((queueKey, partitions) -> {
             if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) {
                 var consumer = getConsumer(queueKey).orElseGet(() -> {
                     Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName());

application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@@ -33,11 +33,14 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
 import org.thingsboard.server.queue.TbQueueConsumer;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.common.consumer.QueueEvent;
+import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask;
+import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask;
 import org.thingsboard.server.queue.discovery.QueueKey;
 import org.thingsboard.server.service.queue.TbMsgPackCallback;
 import org.thingsboard.server.service.queue.TbMsgPackProcessingContext;
 import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats;
-import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager;
+import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager;
 import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
 import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
 import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy;

application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@@ -251,7 +251,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
         }
         log.trace("[{}][{}] On device connect: processing connect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastConnectTime);
         stateData.getState().setLastConnectTime(lastConnectTime);
-        save(deviceId, LAST_CONNECT_TIME, lastConnectTime);
+        save(tenantId, deviceId, LAST_CONNECT_TIME, lastConnectTime);
         pushRuleEngineMessage(stateData, TbMsgType.CONNECT_EVENT);
         checkAndUpdateState(deviceId, stateData);
     }
@@ -271,14 +271,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
     void updateActivityState(DeviceId deviceId, DeviceStateData stateData, long lastReportedActivity) {
         log.trace("updateActivityState - fetched state {} for device {}, lastReportedActivity {}", stateData, deviceId, lastReportedActivity);
         if (stateData != null) {
-            save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
+            save(stateData.getTenantId(), deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
             DeviceState state = stateData.getState();
             state.setLastActivityTime(lastReportedActivity);
             if (!state.isActive()) {
                 state.setActive(true);
                 if (lastReportedActivity <= state.getLastInactivityAlarmTime()) {
                     state.setLastInactivityAlarmTime(0);
-                    save(deviceId, INACTIVITY_ALARM_TIME, 0);
+                    save(stateData.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0);
                 }
                 onDeviceActivityStatusChange(deviceId, true, stateData);
             }
@@ -307,7 +307,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
         }
         log.trace("[{}][{}] On device disconnect: processing disconnect event with ts [{}].", tenantId.getId(), deviceId.getId(), lastDisconnectTime);
         stateData.getState().setLastDisconnectTime(lastDisconnectTime);
-        save(deviceId, LAST_DISCONNECT_TIME, lastDisconnectTime);
+        save(tenantId, deviceId, LAST_DISCONNECT_TIME, lastDisconnectTime);
         pushRuleEngineMessage(stateData, TbMsgType.DISCONNECT_EVENT);
     }
 
@@ -418,7 +418,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
     private void initializeActivityState(DeviceId deviceId, DeviceStateData fetchedState) {
         DeviceStateData cachedState = deviceStates.putIfAbsent(fetchedState.getDeviceId(), fetchedState);
         boolean activityState = Objects.requireNonNullElse(cachedState, fetchedState).getState().isActive();
-        save(deviceId, ACTIVITY_STATE, activityState);
+        save(fetchedState.getTenantId(), deviceId, ACTIVITY_STATE, activityState);
     }
 
     @Override
@@ -496,7 +496,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
                 updateActivityState(deviceId, state, deviceState.getLastActivityTime());
                 if (deviceState.getLastInactivityAlarmTime() != 0L && deviceState.getLastInactivityAlarmTime() >= deviceState.getLastActivityTime()) {
                     deviceState.setLastInactivityAlarmTime(0L);
-                    save(deviceId, INACTIVITY_ALARM_TIME, 0L);
+                    save(state.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0L);
                 }
             }
         }
@@ -583,7 +583,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
         DeviceState state = stateData.getState();
         state.setActive(false);
         state.setLastInactivityAlarmTime(ts);
-        save(deviceId, INACTIVITY_ALARM_TIME, ts);
+        save(stateData.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, ts);
         onDeviceActivityStatusChange(deviceId, false, stateData);
     }
 
@@ -611,7 +611,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
     }
 
     private void onDeviceActivityStatusChange(DeviceId deviceId, boolean active, DeviceStateData stateData) {
-        save(deviceId, ACTIVITY_STATE, active);
+        save(stateData.getTenantId(), deviceId, ACTIVITY_STATE, active);
         pushRuleEngineMessage(stateData, active ? TbMsgType.ACTIVITY_EVENT : TbMsgType.INACTIVITY_EVENT);
         TbMsgMetaData metaData = stateData.getMetaData();
         notificationRuleProcessor.process(DeviceActivityTrigger.builder()
@@ -874,18 +874,18 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
         }
     }
 
-    private void save(DeviceId deviceId, String key, long value) {
-        save(deviceId, new LongDataEntry(key, value), getCurrentTimeMillis());
+    private void save(TenantId tenantId, DeviceId deviceId, String key, long value) {
+        save(tenantId, deviceId, new LongDataEntry(key, value), getCurrentTimeMillis());
     }
 
-    private void save(DeviceId deviceId, String key, boolean value) {
-        save(deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis());
+    private void save(TenantId tenantId, DeviceId deviceId, String key, boolean value) {
+        save(tenantId, deviceId, new BooleanDataEntry(key, value), getCurrentTimeMillis());
     }
 
-    private void save(DeviceId deviceId, KvEntry kvEntry, long ts) {
+    private void save(TenantId tenantId, DeviceId deviceId, KvEntry kvEntry, long ts) {
         if (persistToTelemetry) {
             tsSubService.saveTimeseriesInternal(TimeseriesSaveRequest.builder()
-                    .tenantId(TenantId.SYS_TENANT_ID)
+                    .tenantId(tenantId)
                     .entityId(deviceId)
                     .entry(new BasicTsKvEntry(ts, kvEntry))
                     .ttl(telemetryTtl)
@@ -893,7 +893,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
                     .build());
         } else {
             tsSubService.saveAttributes(AttributesSaveRequest.builder()
-                    .tenantId(TenantId.SYS_TENANT_ID)
+                    .tenantId(tenantId)
                     .entityId(deviceId)
                     .scope(AttributeScope.SERVER_SCOPE)
                     .entry(new BaseAttributeKvEntry(ts, kvEntry))
@@ -925,4 +925,5 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
             log.warn("[{}] Failed to update entry {}", deviceId, kvEntry, t);
         }
     }
+
 }

application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java

@@ -0,0 +1,224 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync.tenant;
+
+import jakarta.annotation.PostConstruct;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.ObjectType;
+import org.thingsboard.server.common.data.Tenant;
+import org.thingsboard.server.common.data.audit.AuditLog;
+import org.thingsboard.server.common.data.event.Event;
+import org.thingsboard.server.common.data.event.EventType;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.HasId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.edqs.AttributeKv;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
+import org.thingsboard.server.common.data.edqs.LatestTsKv;
+import org.thingsboard.server.common.data.kv.TsKvEntry;
+import org.thingsboard.server.common.data.page.PageDataIterable;
+import org.thingsboard.server.common.data.page.TimePageLink;
+import org.thingsboard.server.common.data.relation.EntityRelation;
+import org.thingsboard.server.dao.TenantEntityDao;
+import org.thingsboard.server.dao.attributes.AttributesDao;
+import org.thingsboard.server.dao.audit.AuditLogDao;
+import org.thingsboard.server.dao.entity.EntityDaoRegistry;
+import org.thingsboard.server.dao.event.EventDao;
+import org.thingsboard.server.dao.model.ModelConstants;
+import org.thingsboard.server.dao.relation.RelationDao;
+import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
+import org.thingsboard.server.dao.tenant.TenantDao;
+import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+
+import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV;
+import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG;
+import static org.thingsboard.server.common.data.ObjectType.EVENT;
+import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV;
+import static org.thingsboard.server.common.data.ObjectType.RELATION;
+import static org.thingsboard.server.common.data.ObjectType.TENANT;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class TenantExportService {
+
+    private final EntityDaoRegistry entityDaoRegistry;
+    private final TenantDao tenantDao;
+    private final EventDao eventDao;
+    private final AuditLogDao auditLogDao;
+    private final AttributesDao attributesDao;
+    private final RelationDao relationDao;
+    private final TimeseriesLatestDao timeseriesLatestDao;
+    private final SqlPartitioningRepository partitioningRepository;
+
+    private Map<ObjectType, BiConsumer<TenantId, BiConsumer<ObjectType, Object>>> customExporters;
+    private Map<ObjectType, Exporter> relatedEntitiesExporters;
+
+    private static final Set<ObjectType> RELATED = EnumSet.of(EVENT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV);
+
+    @PostConstruct
+    private void init() {
+        relatedEntitiesExporters = Map.of(
+                RELATION, this::exportRelations,
+                EVENT, this::exportEvents, // todo: query by tenant
+                ATTRIBUTE_KV, this::exportAttributes,
+                LATEST_TS_KV, this::exportLatestTelemetry
+        );
+        customExporters = Map.of(
+                AUDIT_LOG, this::exportAuditLogs
+        );
+    }
+
+    public void exportTenant(TenantId tenantId, ExportConfig config, BiConsumer<ObjectType, Object> processor) {
+        log.info("[{}] Exporting tenant", tenantId);
+        Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId());
+        if (tenant == null) {
+            throw new IllegalArgumentException("Tenant with id " + tenantId + " not found");
+        }
+
+        Set<ObjectType> objectTypes = config.getIncludedObjectTypes();
+        if (objectTypes.contains(TENANT)) {
+            exportEntity(tenantId, TENANT, tenant, config, processor);
+        }
+
+        for (ObjectType type : objectTypes) {
+            if (RELATED.contains(type) || type == TENANT) {
+                continue;
+            }
+            log.debug("[{}] Exporting {} entities", tenantId, type);
+            if (!customExporters.containsKey(type)) {
+                TenantEntityDao<?> dao = entityDaoRegistry.getTenantEntityDao(type);
+                var entities = new PageDataIterable<>(pageLink -> dao.findAllByTenantId(tenantId, pageLink), 100);
+                for (Object entity : entities) {
+                    exportEntity(tenantId, type, entity, config, processor);
+                }
+            } else {
+                customExporters.get(type).accept(tenantId, processor);
+            }
+        }
+    }
+
+    private void exportEntity(TenantId tenantId, ObjectType type, Object entity, ExportConfig config, BiConsumer<ObjectType, Object> processor) {
+        processor.accept(type, entity);
+        if (entity instanceof HasId<?> hasId && hasId.getId() instanceof EntityId entityId) {
+            relatedEntitiesExporters.forEach((relatedEntityType, exporter) -> {
+                if (config.getIncludedObjectTypes().contains(relatedEntityType)) {
+                    exporter.export(tenantId, entityId, processor);
+                }
+            });
+        }
+    }
+
+    private Map<Long, Long> getPartitions(String table) {
+        List<Long> partitionsStartTime = partitioningRepository.fetchPartitions(table).stream().sorted().toList();
+        if (partitionsStartTime.isEmpty()) {
+            return Collections.emptyMap();
+        }
+
+        Map<Long, Long> partitions = new HashMap<>();
+        for (int i = 0; i < partitionsStartTime.size(); i++) {
+            Long startTime = partitionsStartTime.get(i);
+            Long endTime;
+            if (partitionsStartTime.size() - 1 == i) {
+                endTime = System.currentTimeMillis();
+            } else {
+                endTime = partitionsStartTime.get(i + 1) - 1;
+            }
+            partitions.put(startTime, endTime);
+        }
+        return partitions;
+    }
+
+    private void exportAuditLogs(TenantId tenantId, BiConsumer<ObjectType, Object> processor) {
+        Map<Long, Long> partitions = getPartitions(ModelConstants.AUDIT_LOG_TABLE_NAME);
+        partitions.forEach((startTime, endTime) -> {
+            PageDataIterable<AuditLog> auditLogs = new PageDataIterable<>(pageLink -> {
+                return auditLogDao.findAuditLogsByTenantId(tenantId.getId(), null, new TimePageLink(pageLink, startTime, endTime));
+            }, 512);
+            for (AuditLog auditLog : auditLogs) {
+                processor.accept(AUDIT_LOG, auditLog);
+            }
+        });
+    }
+
+    private void exportAttributes(TenantId tenantId, EntityId entityId, BiConsumer<ObjectType, Object> processor) {
+        for (AttributeScope attributeScope : AttributeScope.values()) {
+            List<AttributeKvEntry> attributes = attributesDao.findAll(tenantId, entityId, attributeScope);
+            for (AttributeKvEntry entry : attributes) {
+                AttributeKv attributeKv = new AttributeKv(entityId, attributeScope, entry, entry.getVersion());
+                processor.accept(ATTRIBUTE_KV, attributeKv);
+            }
+        }
+    }
+
+    private void exportRelations(TenantId tenantId, EntityId entityId, BiConsumer<ObjectType, Object> processor) {
+        List<EntityRelation> relations = relationDao.findAllByFrom(tenantId, entityId);
+        for (EntityRelation relation : relations) {
+            processor.accept(RELATION, relation);
+        }
+    }
+
+    @SneakyThrows
+    private void exportLatestTelemetry(TenantId tenantId, EntityId entityId, BiConsumer<ObjectType, Object> processor) {
+        List<TsKvEntry> latestTelemetry = timeseriesLatestDao.findAllLatest(tenantId, entityId).get(30, TimeUnit.SECONDS);
+        for (TsKvEntry tsKvEntry : latestTelemetry) {
+            LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvEntry, tsKvEntry.getVersion());
+            processor.accept(LATEST_TS_KV, latestTsKv);
+        }
+    }
+
+    private void exportEvents(TenantId tenantId, EntityId entityId, BiConsumer<ObjectType, Object> processor) {
+        for (EventType eventType : EventType.values()) {
+            Map<Long, Long> partitions = getPartitions(eventType.getTable());
+            partitions.forEach((startTime, endTime) -> {
+                PageDataIterable<? extends Event> events = new PageDataIterable<>(pageLink -> {
+                    return eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, new TimePageLink(pageLink, startTime, endTime));
+                }, 512);
+                for (Event event : events) {
+                    processor.accept(EVENT, event);
+                }
+            });
+        }
+    }
+
+    private interface Exporter {
+
+        void export(TenantId tenantId, EntityId entityId, BiConsumer<ObjectType, Object> processor);
+
+    }
+
+    @Data
+    public static class ExportConfig {
+
+        private Set<ObjectType> includedObjectTypes;
+
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java

@@ -93,7 +93,8 @@ public class TbCoreTransportApiService {
     @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
     public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
         log.info("Received application ready event. Starting polling for events.");
-        transportApiTemplate.init(transportApiService);
+        transportApiTemplate.subscribe();
+        transportApiTemplate.launch(transportApiService);
     }
 
     @PreDestroy

application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java

@@ -229,6 +229,7 @@ public class DefaultWebSocketService implements WebSocketService {
             } catch (TbRateLimitsException e) {
                 log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e);
             } catch (Exception e) {
+                sendError(sessionRef, cmd.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR, e.getMessage());
                 log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e);
             }
         }

application/src/main/resources/thingsboard.yml

@@ -1615,6 +1615,12 @@ queue:
       edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
       # Kafka properties for Edge event topic
       edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
+      # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions
+      edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}"
+      # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions
+      edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}"
+      # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions
+      edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}"
     consumer-stats:
       # Prints lag between consumer group offset and last messages offset in Kafka topics
       enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
@@ -1688,6 +1694,23 @@ queue:
         enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}"
         # Statistics printing interval for Housekeeper
         print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}"
+  edqs:
+    sync_enabled: "${TB_EDQS_SYNC_ENABLED:true}" # FIXME: disable by default before release
+    api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release
+    mode: "${TB_EDQS_MODE:local}" # local or remote
+    local:
+      rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:/tmp/edqs-backup}"
+    partitions: "${TB_EDQS_PARTITIONS:12}"
+    requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}"
+    responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}"
+    poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}"
+    max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}"
+    max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}"
+    stats:
+      # Enable/disable statistics for EDQS service
+      enabled: "${TB_EDQS_STATS_ENABLED:true}"
+      # Statistics printing interval for EDQS
+      print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}"
 
   vc:
     # Default topic name

application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java

@@ -0,0 +1,67 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.controller;
+
+import org.junit.Before;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.TestPropertySource;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.query.EntityCountQuery;
+import org.thingsboard.server.common.data.query.EntityData;
+import org.thingsboard.server.common.data.query.EntityDataQuery;
+import org.thingsboard.server.common.msg.edqs.EdqsService;
+import org.thingsboard.server.dao.service.DaoSqlTest;
+import org.thingsboard.server.edqs.util.EdqsRocksDb;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.awaitility.Awaitility.await;
+
+@DaoSqlTest
+@TestPropertySource(properties = {
+        "queue.type=kafka", // uncomment to use Kafka
+        "queue.kafka.bootstrap.servers=10.7.1.254:9092",
+        "queue.edqs.sync_enabled=true",
+        "queue.edqs.api_enabled=true",
+        "queue.edqs.mode=local"
+})
+public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest {
+
+    @Autowired
+    private EdqsService edqsService;
+
+    @MockBean
+    private EdqsRocksDb edqsRocksDb;
+
+    @Before
+    public void before() {
+        await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled());
+    }
+
+    @Override
+    protected PageData<EntityData> findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) {
+        return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(query),
+                result -> result.getTotalElements() == expectedResultSize);
+    }
+
+    @Override
+    protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) {
+        return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query),
+                result -> result == expectedResult);
+    }
+
+}

application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@@ -18,12 +18,14 @@ package org.thingsboard.server.controller;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import org.awaitility.Awaitility;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.web.servlet.ResultActions;
+import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.data.Customer;
 import org.thingsboard.server.common.data.Dashboard;
@@ -49,6 +51,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery;
 import org.thingsboard.server.common.data.query.EntityDataSortOrder;
 import org.thingsboard.server.common.data.query.EntityKey;
 import org.thingsboard.server.common.data.query.EntityKeyType;
+import org.thingsboard.server.common.data.query.EntityKeyValueType;
 import org.thingsboard.server.common.data.query.EntityListFilter;
 import org.thingsboard.server.common.data.query.EntityTypeFilter;
 import org.thingsboard.server.common.data.query.FilterPredicateValue;
@@ -73,6 +76,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 @DaoSqlTest
@@ -130,36 +134,25 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         filter.setDeviceNameFilter("");
 
         EntityCountQuery countQuery = new EntityCountQuery(filter);
-
-        Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(97, count.longValue());
+        countByQueryAndCheck(countQuery, 97);
 
         filter.setDeviceTypes(List.of("unknown"));
-        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(0, count.longValue());
+        countByQueryAndCheck(countQuery, 0);
 
         filter.setDeviceTypes(List.of("default"));
         filter.setDeviceNameFilter("Device1");
-
-        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(11, count.longValue());
+        countByQueryAndCheck(countQuery, 11);
 
         EntityListFilter entityListFilter = new EntityListFilter();
         entityListFilter.setEntityType(EntityType.DEVICE);
         entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList()));
-
         countQuery = new EntityCountQuery(entityListFilter);
-
-        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(97, count.longValue());
+        countByQueryAndCheck(countQuery, 97);
 
         EntityTypeFilter filter2 = new EntityTypeFilter();
         filter2.setEntityType(EntityType.DEVICE);
-
-        EntityCountQuery countQuery2 = new EntityCountQuery(filter2);
-
-        Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class);
-        Assert.assertEquals(97, count2.longValue());
+        countQuery = new EntityCountQuery(filter2);
+        countByQueryAndCheck(countQuery, 97);
     }
 
     @Test
@@ -169,14 +162,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         EntityTypeFilter allDeviceFilter = new EntityTypeFilter();
         allDeviceFilter.setEntityType(EntityType.DEVICE);
         EntityCountQuery query = new EntityCountQuery(allDeviceFilter);
-        Long initialCount = doPostWithResponse("/api/entitiesQuery/count", query, Long.class);
+        Long initialCount = countByQuery(query);
 
         loginTenantAdmin();
 
         List<Device> devices = new ArrayList<>();
+        String devicePrefix = "Device" + RandomStringUtils.random(5);
         for (int i = 0; i < 97; i++) {
             Device device = new Device();
-            device.setName("Device" + i);
+            device.setName(devicePrefix + i);
             device.setType("default");
             device.setLabel("testLabel" + (int) (Math.random() * 1000));
             devices.add(doPost("/api/device", device, Device.class));
@@ -189,31 +183,23 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         loginSysAdmin();
 
         EntityCountQuery countQuery = new EntityCountQuery(filter);
-
-        Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(97, count.longValue());
+        countByQueryAndCheck(countQuery, initialCount + 97);
 
         filter.setDeviceType("unknown");
-        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(0, count.longValue());
+        countByQueryAndCheck(countQuery, 0);
 
         filter.setDeviceType("default");
-        filter.setDeviceNameFilter("Device1");
-
-        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(11, count.longValue());
+        filter.setDeviceNameFilter(devicePrefix + "1");
+        countByQueryAndCheck(countQuery, 11);
 
         EntityListFilter entityListFilter = new EntityListFilter();
         entityListFilter.setEntityType(EntityType.DEVICE);
         entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList()));
 
         countQuery = new EntityCountQuery(entityListFilter);
+        countByQueryAndCheck(countQuery, 97);
 
-        count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(97, count.longValue());
-
-        Long count2 = doPostWithResponse("/api/entitiesQuery/count", query, Long.class);
-        Assert.assertEquals(initialCount + 97, count2.longValue());
+        countByQueryAndCheck(query, initialCount + 97);
     }
 
     @Test
@@ -371,11 +357,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
 
-        PageData<EntityData> data =
-                doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-                });
-
-        Assert.assertEquals(97, data.getTotalElements());
+        PageData<EntityData> data = findByQueryAndCheck(query, 97);
         Assert.assertEquals(10, data.getTotalPages());
         Assert.assertTrue(data.hasNext());
         Assert.assertEquals(10, data.getData().size());
@@ -383,8 +365,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         List<EntityData> loadedEntities = new ArrayList<>(data.getData());
         while (data.hasNext()) {
             query = query.next();
-            data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-            });
+            data = findByQuery(query);
             loadedEntities.addAll(data.getData());
         }
         Assert.assertEquals(97, loadedEntities.size());
@@ -406,8 +387,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder);
         query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
-        data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-        });
+        data = findByQuery(query);
         Assert.assertEquals(11, data.getTotalElements());
         Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
 
@@ -423,9 +403,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null);
 
-        PageData<EntityData> data2 =
-                doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference<PageData<EntityData>>() {
-                });
+        PageData<EntityData> data2 = findByQuery(query2);
 
         Assert.assertEquals(97, data2.getTotalElements());
         Assert.assertEquals(10, data2.getTotalPages());
@@ -473,20 +451,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
 
         EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
-
-        PageData<EntityData> data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
-
-        Assert.assertEquals(87, data.getTotalElements());
+        findByQueryAndCheck(query, 87);
 
         filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false)));
         query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
-        data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
-        Assert.assertEquals(10, data.getTotalElements());
+        findByQueryAndCheck(query, 10);
 
         filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true)));
         query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
-        data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
-        Assert.assertEquals(87, data.getTotalElements());
+        findByQueryAndCheck(query, 87);
     }
 
     private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) {
@@ -531,14 +504,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
 
         EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
-        PageData<EntityData> data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-        });
+        PageData<EntityData> data = findByQueryAndCheck(query, 67);
 
         List<EntityData> loadedEntities = new ArrayList<>(data.getData());
         while (data.hasNext()) {
             query = query.next();
-            data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-            });
+            data = findByQuery(query);
             loadedEntities.addAll(data.getData());
         }
         Assert.assertEquals(67, loadedEntities.size());
@@ -551,6 +522,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
         KeyFilter highTemperatureFilter = new KeyFilter();
         highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature"));
+        highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
         NumericFilterPredicate predicate = new NumericFilterPredicate();
         predicate.setValue(FilterPredicateValue.fromDouble(45));
         predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
@@ -559,13 +531,11 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
 
-        data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-        });
+        data = findByQuery(query);
         loadedEntities = new ArrayList<>(data.getData());
         while (data.hasNext()) {
             query = query.next();
-            data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-            });
+            data = findByQuery(query);
             loadedEntities.addAll(data.getData());
         }
         Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
@@ -604,6 +574,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         KeyFilter highTemperatureFilter = new KeyFilter();
         highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime"));
+        highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC);
         NumericFilterPredicate predicate = new NumericFilterPredicate();
 
         DynamicValue<Double> dynamicValue =
@@ -627,16 +598,16 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
 
-        Awaitility.await()
+        await()
                 .alias("data by query")
                 .atMost(TIMEOUT, TimeUnit.SECONDS)
                 .until(() -> {
-                    var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
+                    var data = findByQuery(query);
                     var loadedEntities = new ArrayList<>(data.getData());
                     return loadedEntities.size() == numOfDevices;
                 });
 
-        var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
+        var data = findByQuery(query);
         var loadedEntities = new ArrayList<>(data.getData());
 
         Assert.assertEquals(numOfDevices, loadedEntities.size());
@@ -694,11 +665,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null);
 
-        PageData<EntityData> data =
-                doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {
-                });
-
-        Assert.assertEquals(97, data.getTotalElements());
+        PageData<EntityData> data = findByQueryAndCheck(query, 97);
         Assert.assertEquals(10, data.getTotalPages());
         Assert.assertTrue(data.hasNext());
         Assert.assertEquals(10, data.getData().size());
@@ -712,9 +679,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         });
 
         EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter);
-
-        Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
-        Assert.assertEquals(97, count.longValue());
+        countByQueryAndCheck(countQuery, 97);
     }
 
     @Test
@@ -742,28 +707,28 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30);
         KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME);
         KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName");
-        KeyFilter tenantOwnerTypeFilter =  getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
+        KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
         KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER");
 
         // all devices with ownerName = TEST TENANT
-        EntityCountQuery query = new EntityCountQuery(filter,  List.of(activeAlarmTimeFilter, tenantOwnerNameFilter));
-        checkEntitiesCount(query, numOfDevices);
+        EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter));
+        countByQueryAndCheck(query, numOfDevices);
 
         // all devices with ownerName = TEST TENANT
-        EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter,  List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter));
-        checkEntitiesCount(activeAlarmTimeToLongQuery, 0);
+        EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter));
+        countByQueryAndCheck(activeAlarmTimeToLongQuery, 0);
 
         // all devices with wrong ownerName
         EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter));
-        checkEntitiesCount(wrongTenantNameQuery, 0);
+        countByQueryAndCheck(wrongTenantNameQuery, 0);
 
         // all devices with owner type = TENANT
         EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter));
-        checkEntitiesCount(tenantEntitiesQuery, numOfDevices);
+        countByQueryAndCheck(tenantEntitiesQuery, numOfDevices);
 
         // all devices with owner type = CUSTOMER
         EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter));
-        checkEntitiesCount(customerEntitiesQuery, 0);
+        countByQueryAndCheck(customerEntitiesQuery, 0);
     }
 
     @Test
@@ -790,7 +755,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
         KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5);
         KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME);
         KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName");
-        KeyFilter tenantOwnerTypeFilter =  getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
+        KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT");
         KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER");
 
         EntityDataSortOrder sortOrder = new EntityDataSortOrder(
@@ -874,18 +839,18 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
     }
 
     private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception {
-        Awaitility.await()
+        await()
                 .alias("data by query")
                 .atMost(30, TimeUnit.SECONDS)
                 .until(() -> {
-                    var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
+                    var data = findByQuery(query);
                     var loadedEntities = new ArrayList<>(data.getData());
                     return loadedEntities.size() == expectedNumOfDevices;
                 });
          if (expectedNumOfDevices == 0) {
              return;
          }
-        var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<PageData<EntityData>>() {});
+        var data = findByQuery(query);
         var loadedEntities = new ArrayList<>(data.getData());
 
         Assert.assertEquals(expectedNumOfDevices, loadedEntities.size());
@@ -898,25 +863,36 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
             String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue();
 
             Assert.assertEquals("Device" + i, name);
-            Assert.assertEquals( expectedOwnerName, ownerName);
-            Assert.assertEquals( expectedOwnerType, ownerType);
+            Assert.assertEquals(expectedOwnerName, ownerName);
+            Assert.assertEquals(expectedOwnerType, ownerType);
             Assert.assertEquals("1" + i, alarmActiveTime);
         }
     }
 
-    private void checkEntitiesCount(EntityCountQuery query, int expectedNumOfDevices) {
-        Awaitility.await()
-                .alias("count by query")
-                .atMost(30, TimeUnit.SECONDS)
-                .until(() -> {
-                    var count = doPost("/api/entitiesQuery/count", query, Integer.class);
-                    return count == expectedNumOfDevices;
-                });
-   }
+    protected PageData<EntityData> findByQuery(EntityDataQuery query) throws Exception {
+        return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {});
+    }
+
+    protected PageData<EntityData> findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception {
+        PageData<EntityData> result = findByQuery(query);
+        assertThat(result.getTotalElements()).isEqualTo(expectedResultSize);
+        return result;
+    }
+
+    protected Long countByQuery(EntityCountQuery countQuery) throws Exception {
+        return doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
+    }
+
+    protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) throws Exception {
+        Long result = countByQuery(query);
+        assertThat(result).isEqualTo(expectedResult);
+        return result;
+    }
 
     private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) {
         KeyFilter tenantOwnerNameFilter = new KeyFilter();
         tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName));
+        tenantOwnerNameFilter.setValueType(EntityKeyValueType.STRING);
         StringFilterPredicate ownerNamePredicate = new StringFilterPredicate();
         ownerNamePredicate.setValue(FilterPredicateValue.fromString(value));
         ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL);
@@ -927,6 +903,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
     private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) {
         KeyFilter numericFilter = new KeyFilter();
         numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute));
+        numericFilter.setValueType(EntityKeyValueType.NUMERIC);
         NumericFilterPredicate predicate = new NumericFilterPredicate();
         predicate.setValue(FilterPredicateValue.fromDouble(value));
         predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);

application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java

@@ -153,7 +153,7 @@ public class HashPartitionServiceTest {
             for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) {
                 QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId);
                 for (int partition = 0; partition < partitionCount; partition++) {
-                    ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap());
+                    ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()).get(0);
                     String serviceId = serviceInfo.getServiceId();
                     map.put(serviceId, map.get(serviceId) + 1);
                 }
@@ -308,7 +308,7 @@ public class HashPartitionServiceTest {
         partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine));
         verifyPartitionChangeEvent(event -> {
             QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID);
-            return event.getPartitionsMap().get(queueKey).size() == systemQueue.getPartitions();
+            return event.getNewPartitions().get(queueKey).size() == systemQueue.getPartitions();
         });
 
         Mockito.reset(applicationEventPublisher);
@@ -336,14 +336,14 @@ public class HashPartitionServiceTest {
         // expecting event about no partitions for isolated queue key
         verifyPartitionChangeEvent(event -> {
             QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
-            return event.getPartitionsMap().get(queueKey).isEmpty();
+            return event.getNewPartitions().get(queueKey).isEmpty();
         });
 
         partitionService_dedicated.updateQueues(List.of(queueUpdateMsg));
         partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine));
         verifyPartitionChangeEvent(event -> {
             QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
-            return event.getPartitionsMap().get(queueKey).size() == isolatedQueue.getPartitions();
+            return event.getNewPartitions().get(queueKey).size() == isolatedQueue.getPartitions();
         });
 
 
@@ -361,7 +361,7 @@ public class HashPartitionServiceTest {
         partitionService_dedicated.removeQueues(List.of(queueDeleteMsg));
         verifyPartitionChangeEvent(event -> {
             QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId);
-            return event.getPartitionsMap().get(queueKey).isEmpty();
+            return event.getNewPartitions().get(queueKey).isEmpty();
         });
     }
 
@@ -381,12 +381,12 @@ public class HashPartitionServiceTest {
         Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> {
             List<QueueKey> queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10))
                     .map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId))
-                    .limit(100).collect(Collectors.toList());
+                    .limit(100).toList();
 
             for (int partition = 0; partition < 10; partition++) {
-                ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap());
+                ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()).get(0);
                 for (QueueKey queueKey : queues) {
-                    ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap());
+                    ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()).get(0);
                     assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId())
                             .isEqualTo(expectedAssignedRuleEngine);
                 }

application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@@ -0,0 +1,72 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.entitiy;
+
+import org.junit.Before;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.TestPropertySource;
+import org.thingsboard.server.common.data.id.CustomerId;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.permission.MergedUserPermissions;
+import org.thingsboard.server.common.data.query.EntityCountQuery;
+import org.thingsboard.server.common.data.query.EntityData;
+import org.thingsboard.server.common.data.query.EntityDataQuery;
+import org.thingsboard.server.common.msg.edqs.EdqsService;
+import org.thingsboard.server.dao.service.DaoSqlTest;
+import org.thingsboard.server.edqs.util.EdqsRocksDb;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.awaitility.Awaitility.await;
+
+@DaoSqlTest
+@TestPropertySource(properties = {
+        "queue.edqs.sync_enabled=true",
+        "queue.edqs.api_enabled=true",
+        "queue.edqs.mode=local"
+})
+public class EdqsEntityServiceTest extends EntityServiceTest {
+
+    @Autowired
+    private EdqsService edqsService;
+
+    @MockBean
+    private EdqsRocksDb edqsRocksDb;
+
+    @Before
+    public void beforeEach() {
+        await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled());
+    }
+
+    @Override
+    protected PageData<EntityData> findByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityDataQuery query, long expectedResultSize) {
+        return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, permissions, query),
+                result -> result.getTotalElements() == expectedResultSize);
+    }
+
+    @Override
+    protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) {
+        return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), mergedUserPermissionsPE, countQuery, expectedResult);
+    }
+
+    @Override
+    protected long countByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityCountQuery query, int expectedResult) {
+        return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, permissions, query),
+                result -> result == expectedResult);
+    }
+
+}

application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@@ -626,8 +626,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
                 .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted);
         verify(consumer, times(1)).subscribe(any());
         verify(consumer).subscribe(eq(expectedPartitions));
-        verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream()
-                .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()))));
+        verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions)));
         verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval()));
         verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval()));
         verify(consumer, never()).unsubscribe();
@@ -743,9 +742,11 @@ public class TbRuleEngineQueueConsumerManagerTest {
         }
 
         @Override
-        protected void doSubscribe(List<String> topicNames) {
-            log.debug("doSubscribe({})", topicNames);
-            this.topics = topicNames;
+        protected void doSubscribe(Set<TopicPartitionInfo> partitions) {
+            this.topics = partitions.stream()
+                    .map(TopicPartitionInfo::getFullTopicName)
+                    .collect(Collectors.toList());
+            log.debug("doSubscribe({})", topics);
             subscribed = true;
         }
 

application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java

@@ -556,7 +556,7 @@ public class DefaultDeviceStateServiceTest {
                 .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false));
         PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(
                 new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi)
-        ));
+        ), Collections.emptyMap());
         service.onApplicationEvent(event);
         Thread.sleep(100);
     }

application/src/test/resources/application-test.properties

@@ -53,6 +53,9 @@ sql.ttl.audit_logs.ttl=2592000
 sql.edge_events.partition_size=168
 sql.ttl.edge_events.edge_event_ttl=2592000
 
-server.log_controller_error_stack_trace=false
+server.log_controller_error_stack_trace=true
 
 transport.gateway.dashboard.sync.enabled=false
+
+queue.edqs.sync_enabled=false
+queue.edqs.api_enabled=false

application/src/test/resources/logback-test.xml

@@ -9,7 +9,7 @@
 
 <!--    <logger name="org.thingsboard.server.service.subscription" level="TRACE"/>-->
     <logger name="org.thingsboard.server.controller.TbTestWebSocketClient" level="INFO"/>
-    <logger name="org.thingsboard.server" level="WARN"/>
+    <logger name="org.thingsboard.server" level="INFO"/>
     <logger name="org.springframework" level="WARN"/>
     <logger name="org.springframework.boot.test" level="WARN"/>
     <logger name="org.apache.cassandra" level="WARN"/>

common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java

@@ -26,6 +26,8 @@ public interface TbQueueRequestTemplate<Request extends TbQueueMsg, Response ext
 
     ListenableFuture<Response> send(Request request, long timeoutNs);
 
+    ListenableFuture<Response> send(Request request, Integer partition);
+
     void stop();
 
     void setMessagesStats(MessagesStats messagesStats);

common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java

@@ -15,9 +15,17 @@
  */
 package org.thingsboard.server.queue;
 
+import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
+
+import java.util.Set;
+
 public interface TbQueueResponseTemplate<Request extends TbQueueMsg, Response extends TbQueueMsg> {
 
-    void init(TbQueueHandler<Request, Response> handler);
+    void subscribe();
+
+    void subscribe(Set<TopicPartitionInfo> partitions);
+
+    void launch(TbQueueHandler<Request, Response> handler);
 
     void stop();
 }

common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java

@@ -0,0 +1,102 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data;
+
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+
+public enum ObjectType {
+    TENANT,
+    TENANT_PROFILE,
+    CUSTOMER,
+    ADMIN_SETTINGS,
+    QUEUE,
+    RPC,
+    RULE_CHAIN,
+    OTA_PACKAGE,
+    RESOURCE,
+    ROLE,
+    ENTITY_GROUP,
+    DEVICE_GROUP_OTA_PACKAGE,
+    GROUP_PERMISSION,
+    BLOB_ENTITY,
+    SCHEDULER_EVENT,
+    EVENT,
+    RULE_NODE,
+    CONVERTER,
+    INTEGRATION,
+    USER,
+    USER_CREDENTIALS,
+    USER_AUTH_SETTINGS,
+    EDGE,
+    WIDGETS_BUNDLE,
+    WIDGET_TYPE,
+    DASHBOARD,
+    DEVICE_PROFILE,
+    DEVICE,
+    DEVICE_CREDENTIALS,
+    ASSET_PROFILE,
+    ASSET,
+    ENTITY_VIEW,
+    ALARM,
+    ENTITY_ALARM,
+    OAUTH2_CLIENT,
+    OAUTH2_DOMAIN,
+    OAUTH2_MOBILE,
+    USER_SETTINGS,
+    NOTIFICATION_TARGET,
+    NOTIFICATION_TEMPLATE,
+    NOTIFICATION_RULE,
+    WHITE_LABELING,
+    CUSTOM_TRANSLATION,
+    ALARM_COMMENT,
+    ALARM_TYPE,
+    API_USAGE_STATE,
+    QUEUE_STATS,
+
+    AUDIT_LOG,
+    RELATION,
+    ATTRIBUTE_KV,
+    LATEST_TS_KV;
+
+    public static final Set<ObjectType> edqsTenantTypes = EnumSet.of(
+            TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD,
+            RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE,
+            BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS
+    );
+    public static final Set<ObjectType> edqsTypes = new HashSet<>(edqsTenantTypes);
+    public static final Set<ObjectType> edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD,
+            API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV);
+
+    static {
+        edqsTypes.addAll(Arrays.asList(TENANT, ENTITY_GROUP, RELATION, ATTRIBUTE_KV, LATEST_TS_KV));
+    }
+
+    public EntityType toEntityType() {
+        return EntityType.valueOf(name());
+    }
+
+    public static ObjectType fromEntityType(EntityType entityType) {
+        try {
+            return ObjectType.valueOf(entityType.name());
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java

@@ -0,0 +1,27 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.alarm;
+
+import lombok.Data;
+import org.thingsboard.server.common.data.id.TenantId;
+
+@Data
+public class AlarmType {
+
+    private TenantId tenantId;
+    private String type;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java

@@ -0,0 +1,67 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
+import org.thingsboard.server.common.data.kv.KvEntry;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class AttributeKv implements EdqsObject {
+
+    private EntityId entityId;
+    private AttributeScope scope;
+    private String key;
+    private Long version;
+
+    private Long lastUpdateTs; // optional (on deletion)
+    private KvEntry value; // optional (on deletion)
+
+    public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) {
+        this.entityId = entityId;
+        this.scope = scope;
+        this.key = attributeKvEntry.getKey();
+        this.version = version;
+        this.lastUpdateTs = attributeKvEntry.getLastUpdateTs();
+        this.value = attributeKvEntry;
+    }
+
+    public AttributeKv(EntityId entityId, AttributeScope scope, String key, long version) {
+        this.entityId = entityId;
+        this.scope = scope;
+        this.key = key;
+        this.version = version;
+    }
+
+    @Override
+    public String key() {
+        return "a_" + entityId + "_" + scope + "_" + key;
+    }
+
+    @Override
+    public Long version() {
+        return version;
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import org.thingsboard.server.common.data.ObjectType;
+import org.thingsboard.server.common.data.id.TenantId;
+
+@Data
+@AllArgsConstructor
+@Builder
+public class EdqsEvent {
+    
+    private final TenantId tenantId;
+    private final ObjectType objectType;
+    private final EdqsEventType eventType;
+    private final EdqsObject object;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java

@@ -0,0 +1,21 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+public enum EdqsEventType {
+    UPDATED,
+    DELETED
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java

@@ -0,0 +1,28 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public interface EdqsObject {
+
+    @JsonIgnore
+    String key();
+
+    @JsonIgnore
+    Long version();
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java

@@ -0,0 +1,24 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.Data;
+
+@Data
+@JsonIgnoreProperties
+public class EdqsSyncRequest {
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java

@@ -0,0 +1,62 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.EntityFields;
+import org.thingsboard.server.common.data.edqs.fields.EntityIdFields;
+
+import java.util.UUID;
+
+@Data
+@NoArgsConstructor
+public class Entity implements EdqsObject {
+
+    private EntityType type;
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS)
+    private EntityFields fields;
+
+    public Entity(EntityType type) {
+        this.type = type;
+    }
+
+    public Entity(EntityType type, EntityFields fields) {
+        this.type = type;
+        this.fields = fields;
+    }
+
+    public Entity(EntityType entityType, UUID id, long version) {
+        this.type = entityType;
+        this.fields = new EntityIdFields(id, version);
+    }
+
+    @Override
+    public String key() {
+        return "e_" + fields.getId().toString();
+    }
+
+    @Override
+    public Long version() {
+        return fields.getVersion();
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java

@@ -0,0 +1,62 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.common.data.kv.TsKvEntry;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class LatestTsKv implements EdqsObject {
+
+    private EntityId entityId;
+    private String key;
+    private Long version;
+
+    private Long ts; // optional (on deletion)
+    private KvEntry value; // optional (on deletion)
+
+    public LatestTsKv(EntityId entityId, TsKvEntry tsKvEntry, Long version) {
+        this.entityId = entityId;
+        this.key = tsKvEntry.getKey();
+        this.ts = tsKvEntry.getTs();
+        this.version = version != null ? version : 0L;
+        this.value = tsKvEntry;
+    }
+
+    public LatestTsKv(EntityId entityId, String key, Long version) {
+        this.entityId = entityId;
+        this.key = key;
+        this.version = version != null ? version : 0L;
+    }
+
+    public String key() {
+        return "l_" + entityId + "_" + key;
+    }
+
+    @Override
+    public Long version() {
+        return version;
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class ToCoreEdqsMsg {
+
+    private EdqsSyncRequest syncRequest;
+    private Boolean apiEnabled;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder
+public class ToCoreEdqsRequest {
+
+    private EdqsSyncRequest syncRequest;
+    private Boolean apiEnabled;
+
+    @JsonIgnore
+    public ToCoreEdqsMsg toInternalMsg() {
+        return new ToCoreEdqsMsg(syncRequest, apiEnabled);
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java

@@ -0,0 +1,65 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.experimental.SuperBuilder;
+import org.thingsboard.server.common.data.id.CustomerId;
+
+import java.util.UUID;
+
+@Data
+@SuperBuilder
+public class AbstractEntityFields implements EntityFields {
+
+    private UUID id;
+    private long createdTime;
+    private UUID tenantId;
+    private UUID customerId;
+    private String name;
+    private Long version;
+
+    public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) {
+        this.id = id;
+        this.createdTime = createdTime;
+        this.tenantId = tenantId;
+        this.customerId = (customerId != null && customerId != CustomerId.NULL_UUID) ? customerId : null;
+        this.name = name;
+        this.version = version;
+    }
+
+    public AbstractEntityFields() {
+    }
+
+    public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, String name, Long version) {
+        this(id, createdTime, tenantId, null, name, version);
+    }
+
+    public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, Long version) {
+        this(id, createdTime, tenantId, customerId, null, version);
+
+    }
+
+    public AbstractEntityFields(UUID id, long createdTime, String name, Long version) {
+        this(id, createdTime, null, name, version);
+    }
+
+
+    public AbstractEntityFields(UUID id, long createdTime, UUID tenantId) {
+        this(id, createdTime, tenantId, null, null, null);
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+import org.thingsboard.server.common.data.ApiUsageStateValue;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.EntityIdFactory;
+
+import java.util.UUID;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor
+@SuperBuilder
+public class ApiUsageStateFields extends AbstractEntityFields {
+
+    private EntityId entityId;
+    private ApiUsageStateValue transportState;
+    private ApiUsageStateValue dbStorageState;
+    private ApiUsageStateValue reExecState;
+    private ApiUsageStateValue jsExecState;
+    private ApiUsageStateValue tbelExecState;
+    private ApiUsageStateValue emailExecState;
+    private ApiUsageStateValue smsExecState;
+    private ApiUsageStateValue alarmExecState;
+
+    public ApiUsageStateFields(UUID id, long createdTime, UUID tenantId, UUID entityId, String entityType, ApiUsageStateValue transportState, ApiUsageStateValue dbStorageState,
+                               ApiUsageStateValue reExecState, ApiUsageStateValue jsExecState, ApiUsageStateValue tbelExecState,
+                               ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState) {
+        super(id, createdTime, tenantId);
+        this.entityId = (entityType != null && entityId != null) ? EntityIdFactory.getByTypeAndUuid(entityType, entityId) : null;
+        this.transportState = transportState;
+        this.dbStorageState = dbStorageState;
+        this.reExecState = reExecState;
+        this.jsExecState = jsExecState;
+        this.tbelExecState = tbelExecState;
+        this.emailExecState = emailExecState;
+        this.smsExecState = smsExecState;
+        this.alarmExecState = alarmExecState;
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class AssetFields extends AbstractEntityFields implements ProfileAwareFields {
+
+    private String type;
+    private UUID assetProfileId;
+    private String label;
+    private String additionalInfo;
+
+    @JsonIgnore
+    @Override
+    public String getProfileName() {
+        return type;
+    }
+
+    @JsonIgnore
+    @Override
+    public UUID getProfileId() {
+        return assetProfileId;
+    }
+
+    public AssetFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name,
+                       Long version, String type, String label, UUID assetProfileId, JsonNode additionalInfo) {
+        super(id, createdTime, tenantId, customerId, name, version);
+        this.type = type;
+        this.assetProfileId = assetProfileId;
+        this.label = label;
+        this.additionalInfo = getText(additionalInfo);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java

@@ -0,0 +1,35 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class AssetProfileFields extends AbstractEntityFields {
+
+    private boolean isDefault;
+
+    public AssetProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, boolean isDefault) {
+        super(id, createdTime, tenantId, null, name, version);
+        this.isDefault = isDefault;
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java

@@ -0,0 +1,55 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class CustomerFields extends AbstractEntityFields {
+
+    private String additionalInfo;
+    private String country;
+    private String state;
+    private String city;
+    private String address;
+    private String address2;
+    private String zip;
+    private String phone;
+    private String email;
+
+    public CustomerFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo,
+                          String country, String state, String city, String address, String address2, String zip, String phone, String email) {
+        super(id, createdTime, tenantId, name, version);
+        this.additionalInfo = getText(additionalInfo);
+        this.country = country;
+        this.state = state;
+        this.city = city;
+        this.address = address;
+        this.address2 = address2;
+        this.zip = zip;
+        this.phone = phone;
+        this.email = email;
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java

@@ -0,0 +1,33 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class DashboardFields extends AbstractEntityFields {
+
+    public DashboardFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) {
+        super(id, createdTime, tenantId, customerId, name, version);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class DeviceFields extends AbstractEntityFields implements ProfileAwareFields {
+
+    private String label;
+    private String type;
+    private UUID deviceProfileId;
+    private String additionalInfo;
+
+    @JsonIgnore
+    @Override
+    public String getProfileName() {
+        return type;
+    }
+
+    @JsonIgnore
+    @Override
+    public UUID getProfileId() {
+        return deviceProfileId;
+    }
+
+    public DeviceFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, String type,
+                        String label, UUID deviceProfileId, JsonNode additionalInfo) {
+        super(id, createdTime, tenantId, customerId,  name, version);
+        this.label = label;
+        this.type = type;
+        this.deviceProfileId = deviceProfileId;
+        this.additionalInfo = getText(additionalInfo);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+import org.thingsboard.server.common.data.DeviceProfileType;
+
+import java.util.UUID;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class DeviceProfileFields extends AbstractEntityFields {
+
+    private String type;
+    private boolean isDefault;
+
+    public DeviceProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, DeviceProfileType type, boolean isDefault) {
+        super(id, createdTime, tenantId, null, name, version);
+        this.type = type.name();
+        this.isDefault = isDefault;
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java

@@ -0,0 +1,43 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class EdgeFields extends AbstractEntityFields {
+
+    private String type;
+    private String label;
+    private String additionalInfo;
+
+    public EdgeFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version,
+                      String type, String label, JsonNode additionalInfo) {
+        super(id, createdTime, tenantId, customerId, name, version);
+        this.type = type;
+        this.label = label;
+        this.additionalInfo = getText(additionalInfo);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java

@@ -0,0 +1,171 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.thingsboard.server.common.data.id.EntityId;
+
+import java.util.UUID;
+
+public interface EntityFields {
+
+    Logger log = LoggerFactory.getLogger(EntityFields.class);
+
+    default UUID getId() {
+        return null;
+    }
+
+    default UUID getTenantId() {
+        return null;
+    }
+
+    default UUID getCustomerId() {
+        return null;
+    }
+
+    default long getCreatedTime() {
+        return 0;
+    }
+
+    default String getName() {
+        return "";
+    }
+
+    default String getType() {
+        return "";
+    }
+
+    default String getLabel() {
+        return "";
+    }
+
+    default String getAdditionalInfo() {
+        return "";
+    }
+
+    default String getEmail() {
+        return "";
+    }
+
+    default String getCountry() {
+        return "";
+    }
+
+    default String getState() {
+        return "";
+    }
+
+    default String getCity() {
+        return "";
+    }
+
+    default String getAddress() {
+        return "";
+    }
+
+    default String getAddress2() {
+        return "";
+    }
+
+    default String getZip() {
+        return "";
+    }
+
+    default String getPhone() {
+        return "";
+    }
+
+    default String getRegion() {
+        return "";
+    }
+
+    default String getFirstName() {
+        return "";
+    }
+
+    default String getLastName() {
+        return "";
+    }
+
+    default boolean isEdgeTemplate() {
+        return false;
+    }
+
+    default String getConfiguration() {
+        return "";
+    }
+
+    default String getSchedule() {
+        return "";
+    }
+
+    default EntityId getOriginatorId() {
+        return null;
+    }
+
+    default String getQueueName() {
+        return "";
+    }
+
+    default String getServiceId() {
+        return "";
+    }
+
+    default boolean isDefault() {
+        return false;
+    }
+
+    default UUID getOwnerId() {
+        return null;
+    }
+
+    default Long getVersion() {
+        return null;
+    }
+
+    default String getAsString(String key) {
+        return switch (key) {
+            case "createdTime" -> Long.toString(getCreatedTime());
+            case "type" -> getType();
+            case "label" -> getLabel();
+            case "additionalInfo" -> getAdditionalInfo();
+            case "email" -> getEmail();
+            case "country" -> getCountry();
+            case "state" -> getState();
+            case "city" -> getCity();
+            case "address" -> getAddress();
+            case "address2" -> getAddress2();
+            case "zip" -> getZip();
+            case "phone" -> getPhone();
+            case "region" -> getRegion();
+            case "firstName" -> getFirstName();
+            case "lastName" -> getLastName();
+            case "edgeTemplate" -> Boolean.toString(isEdgeTemplate());
+            case "configuration" -> getConfiguration();
+            case "schedule" -> getSchedule();
+            case "originatorId" -> getOriginatorId().getId().toString();
+            case "originatorType" -> getOriginatorId().getEntityType().toString();
+            case "queueName" -> getQueueName();
+            case "serviceId" -> getServiceId();
+            default -> {
+                log.warn("Unknown field '{}'", key);
+                yield null;
+            }
+        };
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class EntityIdFields implements EntityFields {
+
+    private UUID id;
+    private Long version;
+
+    public EntityIdFields(UUID id, Long version) {
+        this.id = id;
+        this.version = version;
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class EntityViewFields extends AbstractEntityFields {
+
+    private String type;
+    private String additionalInfo;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java

@@ -0,0 +1,298 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.thingsboard.server.common.data.ApiUsageState;
+import org.thingsboard.server.common.data.Customer;
+import org.thingsboard.server.common.data.Dashboard;
+import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.DeviceProfile;
+import org.thingsboard.server.common.data.DeviceProfileType;
+import org.thingsboard.server.common.data.EntityView;
+import org.thingsboard.server.common.data.Tenant;
+import org.thingsboard.server.common.data.TenantProfile;
+import org.thingsboard.server.common.data.User;
+import org.thingsboard.server.common.data.asset.Asset;
+import org.thingsboard.server.common.data.asset.AssetProfile;
+import org.thingsboard.server.common.data.edge.Edge;
+import org.thingsboard.server.common.data.id.CustomerId;
+import org.thingsboard.server.common.data.queue.QueueStats;
+import org.thingsboard.server.common.data.rule.RuleChain;
+import org.thingsboard.server.common.data.rule.RuleNode;
+import org.thingsboard.server.common.data.widget.WidgetType;
+import org.thingsboard.server.common.data.widget.WidgetsBundle;
+
+import java.util.UUID;
+
+public class FieldsUtil {
+
+    public static EntityFields toFields(Object entity) {
+        if (entity instanceof Customer customer) {
+            return toFields(customer);
+        } else if (entity instanceof Tenant tenant) {
+            return toFields(tenant);
+        } else if (entity instanceof TenantProfile tenantProfile) {
+            return toFields(tenantProfile);
+        } else if (entity instanceof Device device) {
+            return toFields(device);
+        } else if (entity instanceof Asset asset) {
+            return toFields(asset);
+        } else if (entity instanceof Edge edge) {
+            return toFields(edge);
+        } else if (entity instanceof EntityView entityView) {
+            return toFields(entityView);
+        } else if (entity instanceof User user) {
+            return toFields(user);
+        } else if (entity instanceof Dashboard dashboard) {
+            return toFields(dashboard);
+        } else if (entity instanceof RuleChain ruleChain) {
+            return toFields(ruleChain);
+        } else if (entity instanceof RuleNode ruleNode) {
+            return toFields(ruleNode);
+        } else if (entity instanceof WidgetType widgetType) {
+            return toFields(widgetType);
+        } else if (entity instanceof WidgetsBundle widgetsBundle) {
+            return toFields(widgetsBundle);
+        } else if (entity instanceof DeviceProfile deviceProfile) {
+            return toFields(deviceProfile);
+        } else if (entity instanceof AssetProfile assetProfile) {
+            return toFields(assetProfile);
+        } else if (entity instanceof QueueStats queueStats) {
+            return toFields(queueStats);
+        } else if (entity instanceof ApiUsageState apiUsageState) {
+            return toFields(apiUsageState);
+        } else {
+            throw new IllegalArgumentException("Unsupported entity type: " + entity.getClass().getName());
+        }
+    }
+
+    private static CustomerFields toFields(Customer entity) {
+        return CustomerFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .customerId(getCustomerId(entity.getCustomerId()))
+                .name(entity.getTitle())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .email(entity.getEmail())
+                .country(entity.getCountry())
+                .state(entity.getState())
+                .city(entity.getCity())
+                .address(entity.getAddress())
+                .address2(entity.getAddress2())
+                .zip(entity.getZip())
+                .phone(entity.getPhone())
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static TenantFields toFields(Tenant entity) {
+        return TenantFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .name(entity.getTitle())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .email(entity.getEmail())
+                .country(entity.getCountry())
+                .state(entity.getState())
+                .city(entity.getCity())
+                .address(entity.getAddress())
+                .address2(entity.getAddress2())
+                .zip(entity.getZip())
+                .phone(entity.getPhone())
+                .region(entity.getRegion())
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static TenantProfileFields toFields(TenantProfile tenantProfile) {
+        return TenantProfileFields.builder()
+                .id(tenantProfile.getUuidId())
+                .createdTime(tenantProfile.getCreatedTime())
+                .name(tenantProfile.getName())
+                .isDefault(tenantProfile.isDefault())
+                .build();
+    }
+
+    private static DeviceFields toFields(Device entity) {
+        return DeviceFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .customerId(getCustomerId(entity.getCustomerId()))
+                .name(entity.getName())
+                .type(entity.getType())
+                .deviceProfileId(entity.getDeviceProfileId().getId())
+                .label(entity.getLabel())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static AssetFields toFields(Asset entity) {
+        return AssetFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .customerId(getCustomerId(entity.getCustomerId()))
+                .name(entity.getName())
+                .type(entity.getType())
+                .assetProfileId(entity.getAssetProfileId().getId())
+                .label(entity.getLabel())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static EdgeFields toFields(Edge entity) {
+        return EdgeFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .customerId(getCustomerId(entity.getCustomerId()))
+                .name(entity.getName())
+                .type(entity.getType())
+                .label(entity.getLabel())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static EntityViewFields toFields(EntityView entity) {
+        return EntityViewFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .customerId(getCustomerId(entity.getCustomerId()))
+                .name(entity.getName())
+                .type(entity.getType())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static UserFields toFields(User entity) {
+        return UserFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .customerId(getCustomerId(entity.getCustomerId()))
+                .firstName(entity.getFirstName())
+                .lastName(entity.getLastName())
+                .email(entity.getEmail())
+                .phone(entity.getPhone())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static DashboardFields toFields(Dashboard entity) {
+        return DashboardFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .customerId(getCustomerId(entity.getCustomerId()))
+                .name(entity.getTitle())
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static RuleChainFields toFields(RuleChain entity) {
+        return RuleChainFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .name(entity.getName())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static RuleNodeFields toFields(RuleNode entity) {
+        return RuleNodeFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .name(entity.getName())
+                .additionalInfo(getText(entity.getAdditionalInfo()))
+                .build();
+    }
+
+    private static WidgetTypeFields toFields(WidgetType entity) {
+        return WidgetTypeFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .name(entity.getName())
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static WidgetsBundleFields toFields(WidgetsBundle entity) {
+        return WidgetsBundleFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .name(entity.getName())
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static AssetProfileFields toFields(DeviceProfile entity) {
+        return AssetProfileFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .name(entity.getName())
+                .isDefault(entity.isDefault())
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static DeviceProfileFields toFields(AssetProfile entity) {
+        return DeviceProfileFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .name(entity.getName())
+                .type(DeviceProfileType.DEFAULT.name())
+                .isDefault(entity.isDefault())
+                .version(entity.getVersion())
+                .build();
+    }
+
+    private static QueueStatsFields toFields(QueueStats entity) {
+        return QueueStatsFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .queueName(entity.getQueueName())
+                .serviceId(entity.getServiceId())
+                .build();
+    }
+
+    private static ApiUsageStateFields toFields(ApiUsageState entity) {
+        return ApiUsageStateFields.builder()
+                .id(entity.getUuidId())
+                .createdTime(entity.getCreatedTime())
+                .entityId(entity.getEntityId())
+                .transportState(entity.getTransportState())
+                .dbStorageState(entity.getDbStorageState())
+                .reExecState(entity.getReExecState())
+                .jsExecState(entity.getJsExecState())
+                .tbelExecState(entity.getTbelExecState())
+                .emailExecState(entity.getEmailExecState())
+                .smsExecState(entity.getSmsExecState())
+                .alarmExecState(entity.getAlarmExecState())
+                .build();
+    }
+
+    public static String getText(JsonNode node) {
+        return node != null ? node.asText() : "";
+    }
+
+    private static UUID getCustomerId(CustomerId customerId) {
+        return (customerId != null && !customerId.getId().equals(CustomerId.NULL_UUID)) ? customerId.getId() : null;
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java

@@ -0,0 +1,41 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class GenericFields extends AbstractEntityFields {
+
+    private String additionalInfo;
+
+    public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) {
+        super(id, createdTime, tenantId, name, version);
+        this.additionalInfo = getText(additionalInfo);
+    }
+
+    public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version) {
+        super(id, createdTime, tenantId, name, version);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java

@@ -0,0 +1,26 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import java.util.UUID;
+
+public interface ProfileAwareFields extends EntityFields {
+
+    String getProfileName();
+
+    UUID getProfileId();
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java

@@ -0,0 +1,42 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class QueueStatsFields extends AbstractEntityFields {
+
+    private String queueName;
+    private String serviceId;
+
+    @Override
+    public String getName() {
+        return queueName + '_' + serviceId;
+    }
+
+    public QueueStatsFields(UUID id, long createdTime, UUID tenantId, String queueName, String serviceId) {
+        super(id, createdTime, tenantId);
+        this.queueName = queueName;
+        this.serviceId = serviceId;
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class RuleChainFields extends AbstractEntityFields {
+
+    private String additionalInfo;
+
+    public RuleChainFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) {
+        super(id, createdTime, tenantId, name, version);
+        this.additionalInfo = getText(additionalInfo);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java

@@ -0,0 +1,43 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class RuleNodeFields implements EntityFields {
+
+    private UUID id;
+    private long createdTime;
+    private String name;
+    private String additionalInfo;
+
+    public RuleNodeFields(UUID id, long createdTime, String name, JsonNode additionalInfo) {
+        this.id = id;
+        this.createdTime = createdTime;
+        this.name = name;
+        this.additionalInfo = getText(additionalInfo);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java

@@ -0,0 +1,62 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class TenantFields extends AbstractEntityFields {
+
+    private String additionalInfo;
+    private String country;
+    private String state;
+    private String city;
+    private String address;
+    private String address2;
+    private String zip;
+    private String phone;
+    private String email;
+    private String region;
+
+    public TenantFields(UUID id, long createdTime, String name, Long version,
+                        JsonNode additionalInfo, String country, String state, String city, String address,
+                        String address2, String zip, String phone, String email, String region) {
+        super(id, createdTime, name, version);
+        this.additionalInfo = getText(additionalInfo);
+        this.country = country;
+        this.state = state;
+        this.city = city;
+        this.address = address;
+        this.address2 = address2;
+        this.zip = zip;
+        this.phone = phone;
+        this.email = email;
+        this.region = region;
+    }
+
+    public TenantFields(UUID id, Long version) {
+        super(id, 0L, null, version);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+import org.thingsboard.server.common.data.id.TenantId;
+
+import java.util.UUID;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class TenantProfileFields extends AbstractEntityFields {
+
+    private boolean isDefault;
+
+    public TenantProfileFields(UUID id, long createdTime, String name, boolean isDefault) {
+        super(id, createdTime, TenantId.SYS_TENANT_ID.getId(), null, name, 0L);
+        this.isDefault = isDefault;
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java

@@ -0,0 +1,48 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText;
+
+@Data
+@NoArgsConstructor
+@SuperBuilder
+public class UserFields extends AbstractEntityFields {
+
+    private String firstName;
+    private String lastName;
+    private String email;
+    private String phone;
+    private String additionalInfo;
+
+    public UserFields(UUID id, long createdTime, UUID tenantId, UUID customerId,
+                      Long version, String firstName, String lastName, String email,
+                      String phone, JsonNode additionalInfo) {
+        super(id, createdTime, tenantId, customerId, version);
+        this.firstName = firstName;
+        this.lastName = lastName;
+        this.email = email;
+        this.phone = phone;
+        this.additionalInfo = getText(additionalInfo);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+@NoArgsConstructor
+@SuperBuilder
+public class WidgetTypeFields extends AbstractEntityFields {
+
+    public WidgetTypeFields(UUID id, long createdTime, UUID tenantId, String name, Long version) {
+        super(id, createdTime, tenantId, name, version);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.fields;
+
+import lombok.NoArgsConstructor;
+import lombok.experimental.SuperBuilder;
+
+import java.util.UUID;
+
+@NoArgsConstructor
+@SuperBuilder
+public class WidgetsBundleFields extends AbstractEntityFields {
+
+    public WidgetsBundleFields(UUID id, long createdTime, UUID tenantId, String name, Long version) {
+        super(id, createdTime, tenantId, name, version);
+    }
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.query;
+
+import com.fasterxml.jackson.annotation.JsonIncludeProperties;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.thingsboard.server.common.data.permission.MergedUserPermissions;
+import org.thingsboard.server.common.data.query.EntityCountQuery;
+import org.thingsboard.server.common.data.query.EntityDataQuery;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+public class EdqsRequest {
+
+    private EntityDataQuery entityDataQuery;
+    private EntityCountQuery entityCountQuery;
+    @JsonIncludeProperties({"genericPermissions", "groupPermissions"})
+    private MergedUserPermissions userPermissions;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java

@@ -0,0 +1,35 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.query;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.query.EntityData;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class EdqsResponse {
+
+    private PageData<EntityData> entityDataQueryResult;
+    private Long entityCountQueryResult;
+    private String error;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java

@@ -0,0 +1,41 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.edqs.query;
+
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.query.EntityData;
+import org.thingsboard.server.common.data.query.EntityKeyType;
+import org.thingsboard.server.common.data.query.TsValue;
+
+import java.util.Collections;
+import java.util.Map;
+
+@Data
+@RequiredArgsConstructor
+public class QueryResult {
+
+    private final EntityId entityId;
+    private final boolean readAttrs;
+    private final boolean readTs;
+    private final Map<EntityKeyType, Map<String, TsValue>> latest;
+
+    public EntityData toOldEntityData() {
+        return new EntityData(entityId, readAttrs, readTs, latest, Collections.emptyMap(), Collections.emptyMap());
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java

@@ -15,11 +15,15 @@
  */
 package org.thingsboard.server.common.data.id;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 import java.util.UUID;
 
 public class UserAuthSettingsId extends UUIDBased {
 
-    public UserAuthSettingsId(UUID id) {
+    @JsonCreator
+    public UserAuthSettingsId(@JsonProperty("id") UUID id) {
         super(id);
     }
 

common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java

@@ -23,6 +23,7 @@ import java.util.NoSuchElementException;
 public abstract class BasePageDataIterable<T> implements Iterable<T>, Iterator<T> {
 
     private final int fetchSize;
+    private SortOrder sortOrder;
 
     private List<T> currentItems;
     private int currentIdx;
@@ -35,6 +36,12 @@ public abstract class BasePageDataIterable<T> implements Iterable<T>, Iterator<T
         this.fetchSize = fetchSize;
     }
 
+    public BasePageDataIterable(int fetchSize, SortOrder sortOrder) {
+        super();
+        this.fetchSize = fetchSize;
+        this.sortOrder = sortOrder;
+    }
+
     @Override
     public Iterator<T> iterator() {
         return this;
@@ -43,7 +50,7 @@ public abstract class BasePageDataIterable<T> implements Iterable<T>, Iterator<T
     @Override
     public boolean hasNext() {
         if (!initialized) {
-            fetch(new PageLink(fetchSize));
+            fetch(new PageLink(fetchSize, 0, null, sortOrder));
             initialized = true;
         }
         if (currentIdx == currentItems.size()) {

common/data/src/main/java/org/thingsboard/server/common/data/page/PageDataIterable.java

@@ -24,6 +24,11 @@ public class PageDataIterable<T> extends BasePageDataIterable<T> {
         this.function = function;
     }
 
+    public PageDataIterable(FetchFunction<T> function, int fetchSize, SortOrder sortOrder) {
+        super(fetchSize, sortOrder);
+        this.function = function;
+    }
+
     @Override
     PageData<T> fetchPageData(PageLink link) {
         return function.fetch(link);

dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java → common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.thingsboard.server.dao.sql.query;
+package org.thingsboard.server.common.data.permission;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -21,8 +21,12 @@ import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.TenantId;
 
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
 @AllArgsConstructor
-public class QuerySecurityContext {
+public class QueryContext {
 
     @Getter
     private final TenantId tenantId;
@@ -33,7 +37,11 @@ public class QuerySecurityContext {
     @Getter
     private final boolean ignorePermissionCheck;
 
-    public QuerySecurityContext(TenantId tenantId, CustomerId customerId, EntityType entityType) {
+    @Getter
+    private final Map<UUID, UUID> relatedParentIdMap = new HashMap<>();
+
+    public QueryContext(TenantId tenantId, CustomerId customerId, EntityType entityType) {
         this(tenantId, customerId, entityType, false);
     }
+
 }

common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java

@@ -15,7 +15,6 @@
  */
 package org.thingsboard.server.common.data.query;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import org.thingsboard.server.common.data.validation.NoXss;
@@ -26,7 +25,6 @@ import java.io.Serializable;
 @RequiredArgsConstructor
 public class DynamicValue<T> implements Serializable {
 
-    @JsonIgnore
     private T resolvedValue;
 
     private final DynamicValueSourceType sourceType;

common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java

@@ -15,6 +15,7 @@
  */
 package org.thingsboard.server.common.data.query;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Getter;
 import lombok.ToString;
@@ -24,6 +25,7 @@ import java.util.List;
 
 @Schema
 @ToString
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class EntityCountQuery {
 
     @Getter

common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java

@@ -16,20 +16,22 @@
 package org.thingsboard.server.common.data.query;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.RequiredArgsConstructor;
+import lombok.NoArgsConstructor;
 import org.thingsboard.server.common.data.id.EntityId;
 
 import java.util.Map;
 
 @Data
-@RequiredArgsConstructor
+@AllArgsConstructor
+@NoArgsConstructor
 public class EntityData {
 
-    private final EntityId entityId;
-    private final Map<EntityKeyType, Map<String, TsValue>> latest;
-    private final Map<String, TsValue[]> timeseries;
-    private final Map<Integer, ComparisonTsValue> aggLatest;
+    private EntityId entityId;
+    private Map<EntityKeyType, Map<String, TsValue>> latest;
+    private Map<String, TsValue[]> timeseries;
+    private Map<Integer, ComparisonTsValue> aggLatest;
 
     public EntityData(EntityId entityId, Map<EntityKeyType, Map<String, TsValue>> latest, Map<String, TsValue[]> timeseries) {
         this(entityId, latest, timeseries, null);
@@ -44,4 +46,5 @@ public class EntityData {
             aggLatest.clear();
         }
     }
+
 }

common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java

@@ -15,10 +15,22 @@
  */
 package org.thingsboard.server.common.data.queue;
 
+import lombok.Data;
+
 public interface QueueConfig {
 
     boolean isConsumerPerPartition();
 
     int getPollInterval();
 
+    static QueueConfig of(boolean consumerPerPartition, long pollInterval) {
+        return new BasicQueueConfig(consumerPerPartition, (int) pollInterval);
+    }
+
+    @Data
+    class BasicQueueConfig implements QueueConfig {
+        private final boolean consumerPerPartition;
+        private final int pollInterval;
+    }
+
 }

common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java

@@ -18,13 +18,15 @@ package org.thingsboard.server.common.data.queue;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import org.thingsboard.server.common.data.BaseData;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.HasEntityType;
 import org.thingsboard.server.common.data.HasTenantId;
 import org.thingsboard.server.common.data.id.QueueStatsId;
 import org.thingsboard.server.common.data.id.TenantId;
 
 @EqualsAndHashCode(callSuper = true)
 @Data
-public class QueueStats extends BaseData<QueueStatsId> implements HasTenantId {
+public class QueueStats extends BaseData<QueueStatsId> implements HasTenantId, HasEntityType {
     private TenantId tenantId;
     private String queueName;
     private String serviceId;
@@ -36,4 +38,8 @@ public class QueueStats extends BaseData<QueueStatsId> implements HasTenantId {
         super(id);
     }
 
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.QUEUE_STATS;
+    }
 }

common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java

@@ -25,6 +25,7 @@ import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
 import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo;
 import org.thingsboard.server.common.data.HasVersion;
+import org.thingsboard.server.common.data.edqs.EdqsObject;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.validation.Length;
 
@@ -34,7 +35,7 @@ import java.io.Serializable;
 @Schema
 @EqualsAndHashCode(exclude = "additionalInfoBytes")
 @ToString(exclude = {"additionalInfoBytes"})
-public class EntityRelation implements HasVersion, Serializable {
+public class EntityRelation implements HasVersion, Serializable, EdqsObject {
 
     private static final long serialVersionUID = 2807343040519543363L;
 
@@ -107,7 +108,7 @@ public class EntityRelation implements HasVersion, Serializable {
         return typeGroup;
     }
 
-    @Schema(description = "Additional parameters of the relation",implementation = com.fasterxml.jackson.databind.JsonNode.class)
+    @Schema(description = "Additional parameters of the relation", implementation = com.fasterxml.jackson.databind.JsonNode.class)
     public JsonNode getAdditionalInfo() {
         return BaseDataWithAdditionalInfo.getJson(() -> additionalInfo, () -> additionalInfoBytes);
     }
@@ -116,4 +117,14 @@ public class EntityRelation implements HasVersion, Serializable {
         BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes);
     }
 
+    @JsonIgnore
+    public String key() {
+        return "r_" + from + "_" + to + "_" + typeGroup + "_" + type;
+    }
+
+    @Override
+    public Long version() {
+        return version;
+    }
+
 }

common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java

@@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.util;
 
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -75,4 +76,11 @@ public class CollectionsUtil {
         return isEmpty(collection) || collection.contains(element);
     }
 
+    public static <V> HashSet<V> concat(Set<V> set1, Set<V> set2) {
+        HashSet<V> result = new HashSet<>();
+        result.addAll(set1);
+        result.addAll(set2);
+        return result;
+    }
+
 }

common/edqs/pom.xml

@@ -0,0 +1,97 @@
+<!--
+
+    Copyright © 2016-2024 ThingsBoard, Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.thingsboard</groupId>
+        <version>4.0.0PE-SNAPSHOT</version>
+        <artifactId>common</artifactId>
+    </parent>
+    <groupId>org.thingsboard.common</groupId>
+    <artifactId>edqs</artifactId>
+    <packaging>jar</packaging>
+
+    <name>Thingsboard Server EDQS API</name>
+    <url>https://thingsboard.io</url>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <main.dir>${basedir}/../..</main.dir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.rocksdb</groupId>
+            <artifactId>rocksdbjni</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>proto</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>data</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>util</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>message</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>stats</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>cluster-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.thingsboard.common</groupId>
+            <artifactId>queue</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.github.ben-manes.caffeine</groupId>
+            <artifactId>caffeine</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context-support</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-autoconfigure</artifactId>
+        </dependency>
+    </dependencies>
+
+    <distributionManagement>
+        <repository>
+            <id>thingsboard-repo-deploy</id>
+            <name>ThingsBoard Repo Deployment</name>
+            <url>https://repo.thingsboard.io/artifactory/libs-release-public</url>
+        </repository>
+    </distributionManagement>
+
+</project>

common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java

@@ -0,0 +1,46 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.ToString;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields;
+
+import java.util.UUID;
+
+@ToString(callSuper = true)
+public class ApiUsageStateData extends BaseEntityData<ApiUsageStateFields> {
+
+    public ApiUsageStateData(UUID entityId) {
+        super(entityId);
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.API_USAGE_STATE;
+    }
+
+    @Override
+    public String getEntityName() {
+        return getEntityOwnerName();
+    }
+
+    @Override
+    public String getEntityOwnerName() {
+        return repo.getOwnerName(fields.getEntityId());
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.ToString;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.AssetFields;
+
+import java.util.UUID;
+
+@ToString(callSuper = true)
+public class AssetData extends ProfileAwareData<AssetFields> {
+
+    public AssetData(UUID id) {
+        super(id);
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.ASSET;
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java

@@ -0,0 +1,180 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.EntityFields;
+import org.thingsboard.server.common.data.id.CustomerId;
+import org.thingsboard.server.common.data.permission.QueryContext;
+import org.thingsboard.server.common.data.query.EntityKeyType;
+import org.thingsboard.server.edqs.data.dp.BoolDataPoint;
+import org.thingsboard.server.edqs.data.dp.DataPoint;
+import org.thingsboard.server.edqs.data.dp.LongDataPoint;
+import org.thingsboard.server.edqs.data.dp.StringDataPoint;
+import org.thingsboard.server.edqs.query.DataKey;
+import org.thingsboard.server.edqs.repo.TenantRepo;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+@ToString
+public abstract class BaseEntityData<T extends EntityFields> implements EntityData<T> {
+
+    @Getter
+    private final UUID id;
+    @Getter
+    protected final Map<Integer, DataPoint> serverAttrMap;
+    @Getter
+    private final Map<Integer, DataPoint> tMap;
+
+    @Getter
+    @Setter
+    private volatile UUID customerId;
+
+    @Setter
+    protected TenantRepo repo;
+
+    @Getter
+    @Setter
+    protected volatile T fields;
+
+    public BaseEntityData(UUID id) {
+        this.id = id;
+        this.serverAttrMap = new ConcurrentHashMap<>();
+        this.tMap = new ConcurrentHashMap<>();
+    }
+
+    @Override
+    public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) {
+        return switch (entityKeyType) {
+            case ATTRIBUTE, SERVER_ATTRIBUTE -> serverAttrMap.get(keyId);
+            default -> null;
+        };
+    }
+
+    @Override
+    public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) {
+        return serverAttrMap.put(keyId, value) == null;
+    }
+
+    @Override
+    public boolean removeAttr(Integer keyId, AttributeScope scope) {
+        return serverAttrMap.remove(keyId) != null;
+    }
+
+    @Override
+    public DataPoint getTs(Integer keyId) {
+        return tMap.get(keyId);
+    }
+
+    @Override
+    public boolean putTs(Integer keyId, DataPoint value) {
+        return tMap.put(keyId, value) == null;
+    }
+
+    @Override
+    public boolean removeTs(Integer keyId) {
+        return tMap.remove(keyId) != null;
+    }
+
+    @Override
+    public EntityType getOwnerType() {
+        return customerId != null ? EntityType.CUSTOMER : EntityType.TENANT;
+    }
+
+    @Override
+    public DataPoint getDataPoint(DataKey key, QueryContext ctx) {
+        return switch (key.type()) {
+            case TIME_SERIES -> getTs(key.keyId());
+            case ATTRIBUTE, SERVER_ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE -> getAttr(key.keyId(), key.type());
+            case ENTITY_FIELD -> getField(key, ctx);
+            default -> throw new RuntimeException(key.type() + " not supported");
+        };
+    }
+
+    private DataPoint getField(DataKey newKey, QueryContext ctx) {
+        if (fields == null) {
+            return null;
+        }
+        String key = newKey.key();
+        return switch (key) {
+            case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime());
+            case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate());
+            case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx));
+            default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false);
+        };
+    }
+
+    @Override
+    public String getField(String name) {
+        if (fields == null) {
+            return null;
+        }
+        return switch (name) {
+            case "name" -> getEntityName();
+            case "ownerName" -> getEntityOwnerName();
+            case "ownerType" -> customerId != null ? EntityType.CUSTOMER.name() : EntityType.TENANT.name();
+            case "entityType" -> Optional.ofNullable(getEntityType()).map(EntityType::name).orElse("");
+            default -> fields.getAsString(name);
+        };
+    }
+
+    public String getEntityOwnerName() {
+        return repo.getOwnerName(getCustomerId() == null || CustomerId.NULL_UUID.equals(getCustomerId()) ? null :
+                new CustomerId(getCustomerId()));
+    }
+
+    public String getEntityName() {
+        return getFields().getName();
+    }
+
+    private String getRelatedParentId(QueryContext ctx) {
+        return Optional.ofNullable(ctx.getRelatedParentIdMap().get(getId()))
+                .map(UUID::toString)
+                .orElse("");
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return null;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return fields == null;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BaseEntityData<?> that = (BaseEntityData<?>) o;
+        return Objects.equals(id, that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java

@@ -0,0 +1,62 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.CustomerFields;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class CustomerData extends BaseEntityData<CustomerFields> {
+
+    private final ConcurrentMap<EntityType, ConcurrentMap<UUID, EntityData<?>>> entitiesById = new ConcurrentHashMap<>();
+
+    public CustomerData(UUID entityId) {
+        super(entityId);
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.CUSTOMER;
+    }
+
+    public Collection<EntityData<?>> getEntities(EntityType entityType) {
+        var map = entitiesById.get(entityType);
+        if (map == null) {
+            return Collections.emptyList();
+        } else {
+            return map.values();
+        }
+    }
+
+    public void addOrUpdate(EntityData<?> ed) {
+        entitiesById.computeIfAbsent(ed.getEntityType(), et -> new ConcurrentHashMap<>()).put(ed.getId(), ed);
+    }
+
+    public boolean remove(EntityData<?> ed) {
+        var map = entitiesById.get(ed.getEntityType());
+        if (map != null) {
+            return map.remove(ed.getId()) != null;
+        } else {
+            return false;
+        }
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java

@@ -0,0 +1,86 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.ToString;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.DeviceFields;
+import org.thingsboard.server.common.data.query.EntityKeyType;
+import org.thingsboard.server.edqs.data.dp.DataPoint;
+
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+@ToString(callSuper = true)
+public class DeviceData extends ProfileAwareData<DeviceFields> {
+
+    private final Map<Integer, DataPoint> clientAttrMap;
+    private final Map<Integer, DataPoint> sharedAttrMap;
+
+    public DeviceData(UUID entityId) {
+        super(entityId);
+        this.clientAttrMap = new ConcurrentHashMap<>();
+        this.sharedAttrMap = new ConcurrentHashMap<>();
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.DEVICE;
+    }
+
+    @Override
+    public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) {
+        return switch (entityKeyType) {
+            case ATTRIBUTE -> getAttributeDataPoint(keyId);
+            case SERVER_ATTRIBUTE -> serverAttrMap.get(keyId);
+            case CLIENT_ATTRIBUTE -> clientAttrMap.get(keyId);
+            case SHARED_ATTRIBUTE -> sharedAttrMap.get(keyId);
+            default -> throw new RuntimeException(entityKeyType + " not implemented");
+        };
+    }
+
+    @Override
+    public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) {
+        return switch (scope) {
+            case SERVER_SCOPE -> serverAttrMap.put(keyId, value) == null;
+            case CLIENT_SCOPE -> clientAttrMap.put(keyId, value) == null;
+            case SHARED_SCOPE -> sharedAttrMap.put(keyId, value) == null;
+        };
+    }
+
+    @Override
+    public boolean removeAttr(Integer keyId, AttributeScope scope) {
+        return switch (scope) {
+            case SERVER_SCOPE -> serverAttrMap.remove(keyId) != null;
+            case CLIENT_SCOPE -> clientAttrMap.remove(keyId) != null;
+            case SHARED_SCOPE -> sharedAttrMap.remove(keyId) != null;
+        };
+    }
+
+    private DataPoint getAttributeDataPoint(Integer keyId) {
+        DataPoint dp = serverAttrMap.get(keyId);
+        if (dp == null) {
+            dp = sharedAttrMap.get(keyId);
+            if (dp == null) {
+                dp = clientAttrMap.get(keyId);
+            }
+        }
+        return dp;
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java

@@ -0,0 +1,65 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.EntityFields;
+import org.thingsboard.server.common.data.permission.QueryContext;
+import org.thingsboard.server.common.data.query.EntityKeyType;
+import org.thingsboard.server.edqs.data.dp.DataPoint;
+import org.thingsboard.server.edqs.query.DataKey;
+import org.thingsboard.server.edqs.repo.TenantRepo;
+
+import java.util.UUID;
+
+public interface EntityData<T extends EntityFields> {
+
+    UUID getId();
+
+    EntityType getEntityType();
+
+    UUID getCustomerId();
+
+    void setCustomerId(UUID customerId);
+
+    void setRepo(TenantRepo repo);
+
+    T getFields();
+
+    void setFields(T fields);
+
+    DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType);
+
+    boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value);
+
+    boolean removeAttr(Integer keyId, AttributeScope scope);
+
+    DataPoint getTs(Integer keyId);
+
+    boolean putTs(Integer keyId, DataPoint value);
+
+    boolean removeTs(Integer keyId);
+
+    EntityType getOwnerType();
+
+    DataPoint getDataPoint(DataKey key, QueryContext queryContext);
+
+    String getField(String name);
+
+    boolean isEmpty();
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields;
+
+import java.util.Collection;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class EntityGroupData extends BaseEntityData<EntityGroupFields> {
+
+    private final ConcurrentMap<UUID, EntityData<?>> entitiesById = new ConcurrentHashMap<>();
+
+    public EntityGroupData(UUID entityId) {
+        super(entityId);
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.ENTITY_GROUP;
+    }
+
+    public Collection<EntityData<?>> getEntities() {
+        return entitiesById.values();
+    }
+
+    public boolean addOrUpdate(EntityData<?> ed) {
+        return entitiesById.put(ed.getId(), ed) == null;
+    }
+
+    public boolean remove(EntityData<?> ed) {
+        return entitiesById.remove(ed.getId()) != null;
+    }
+
+    public EntityData<?> getEntity(UUID entityId) {
+        return entitiesById.get(entityId);
+    }
+
+    public boolean remove(UUID toId) {
+        return entitiesById.remove(toId) != null;
+    }
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java

@@ -0,0 +1,39 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.ToString;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.EntityFields;
+
+import java.util.UUID;
+
+@ToString(callSuper = true)
+public class EntityProfileData extends BaseEntityData<EntityFields> {
+
+    private final EntityType entityType;
+
+    public EntityProfileData(UUID entityId, EntityType entityType) {
+        super(entityId);
+        this.entityType = entityType;
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return entityType;
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.ToString;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.EntityFields;
+
+import java.util.UUID;
+
+@ToString(callSuper = true)
+public class GenericData extends BaseEntityData<EntityFields> {
+
+    private final EntityType entityType;
+
+    public GenericData(EntityType entityType, UUID entityId) {
+        super(entityId);
+        this.entityType = entityType;
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return entityType;
+    }
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java

@@ -0,0 +1,28 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import org.thingsboard.server.common.data.edqs.fields.ProfileAwareFields;
+
+import java.util.UUID;
+
+public abstract class ProfileAwareData<T> extends BaseEntityData<ProfileAwareFields> {
+
+    public ProfileAwareData(UUID id) {
+        super(id);
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java

@@ -0,0 +1,26 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.relation.RelationTypeGroup;
+
+import java.util.UUID;
+
+public record RelationData(UUID fromId, EntityType fromType, UUID toId, EntityType toType, String type,
+                           RelationTypeGroup typeGroup) {
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java

@@ -0,0 +1,26 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.Data;
+
+@Data
+public class RelationInfo {
+
+    private final String type;
+    private final EntityData<?> target;
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java

@@ -0,0 +1,62 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import lombok.NoArgsConstructor;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+@NoArgsConstructor
+public class RelationsRepo {
+
+    private final ConcurrentMap<UUID, Set<RelationInfo>> fromRelations = new ConcurrentHashMap<>();
+    private final ConcurrentMap<UUID, Set<RelationInfo>> toRelations = new ConcurrentHashMap<>();
+
+    public boolean add(EntityData<?> from, EntityData<?> to, String type) {
+        boolean addedFromRelation = fromRelations.computeIfAbsent(from.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, to));
+        boolean addedToRelation = toRelations.computeIfAbsent(to.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, from));
+        return addedFromRelation || addedToRelation;
+    }
+
+    public Set<RelationInfo> getFrom(UUID entityId) {
+        var result = fromRelations.get(entityId);
+        return result == null ? Collections.emptySet() : result;
+    }
+
+    public Set<RelationInfo> getTo(UUID entityId) {
+        var result = toRelations.get(entityId);
+        return result == null ? Collections.emptySet() : result;
+    }
+
+    public boolean remove(UUID from, UUID to, String type) {
+        boolean removedFromRelation = false;
+        boolean removedToRelation = false;
+        Set<RelationInfo> fromRelations = this.fromRelations.get(from);
+        if (fromRelations != null) {
+            removedFromRelation = fromRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(to) && relationInfo.getType().equals(type));
+        }
+        Set<RelationInfo> toRelations = this.toRelations.get(to);
+        if (toRelations != null) {
+            removedToRelation = toRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(from) && relationInfo.getType().equals(type));
+        }
+        return removedFromRelation || removedToRelation;
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data;
+
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.edqs.fields.TenantFields;
+
+import java.util.UUID;
+
+public class TenantData extends BaseEntityData<TenantFields> {
+
+    public TenantData(UUID entityId) {
+        super(entityId);
+    }
+
+    @Override
+    public EntityType getEntityType() {
+        return EntityType.TENANT;
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java

@@ -0,0 +1,56 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data.dp;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public abstract class AbstractDataPoint implements DataPoint {
+
+    @Getter
+    private final long ts;
+
+    @Override
+    public String getStr() {
+        throw new RuntimeException(NOT_SUPPORTED);
+    }
+
+    @Override
+    public long getLong() {
+        throw new RuntimeException(NOT_SUPPORTED);
+    }
+
+    @Override
+    public double getDouble() {
+        throw new RuntimeException(NOT_SUPPORTED);
+    }
+
+    @Override
+    public boolean getBool() {
+        throw new RuntimeException(NOT_SUPPORTED);
+    }
+
+    @Override
+    public String getJson() {
+        throw new RuntimeException(NOT_SUPPORTED);
+    }
+
+    public String toString() {
+        return valueToString();
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java

@@ -0,0 +1,46 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data.dp;
+
+import lombok.Getter;
+import org.thingsboard.server.common.data.kv.DataType;
+
+public class BoolDataPoint extends AbstractDataPoint {
+
+    @Getter
+    private final boolean value;
+
+    public BoolDataPoint(long ts, boolean value) {
+        super(ts);
+        this.value = value;
+    }
+
+    @Override
+    public DataType getType() {
+        return DataType.BOOLEAN;
+    }
+
+    @Override
+    public boolean getBool() {
+        return value;
+    }
+
+    @Override
+    public String valueToString() {
+        return Boolean.toString(value);
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java

@@ -0,0 +1,31 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data.dp;
+
+import org.thingsboard.server.common.data.kv.DataType;
+
+public class CompressedJsonDataPoint extends CompressedStringDataPoint {
+
+    public CompressedJsonDataPoint(long ts, String value) {
+        super(ts, value);
+    }
+
+    @Override
+    public DataType getType() {
+        return DataType.JSON;
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java

@@ -0,0 +1,64 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data.dp;
+
+import lombok.Getter;
+import lombok.SneakyThrows;
+import org.thingsboard.server.common.data.kv.DataType;
+import org.thingsboard.server.edqs.repo.TbBytePool;
+import org.xerial.snappy.Snappy;
+
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class CompressedStringDataPoint extends AbstractDataPoint {
+
+    public static final int MIN_STR_SIZE_TO_COMPRESS = 512;
+    @Getter
+    private final byte[] value;
+
+    public static final AtomicInteger cnt = new AtomicInteger();
+    public static final AtomicLong uncompressedLength = new AtomicLong();
+    public static final AtomicLong compressedLength = new AtomicLong();
+
+    @SneakyThrows
+    public CompressedStringDataPoint(long ts, String value) {
+        super(ts);
+        cnt.incrementAndGet();
+        uncompressedLength.addAndGet(value.getBytes(StandardCharsets.UTF_8).length);
+        this.value = TbBytePool.intern(Snappy.compress(value));
+        compressedLength.addAndGet(this.value.length);
+    }
+
+    @Override
+    public DataType getType() {
+        return DataType.STRING;
+    }
+
+    @SneakyThrows
+    @Override
+    public String getStr() {
+        return Snappy.uncompressString(value);
+    }
+
+    @SneakyThrows
+    @Override
+    public String valueToString() {
+        return Snappy.uncompressString(value);
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java

@@ -0,0 +1,40 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data.dp;
+
+import org.thingsboard.server.common.data.kv.DataType;
+
+public interface DataPoint {
+
+    String NOT_SUPPORTED = "Not supported!";
+
+    long getTs();
+
+    DataType getType();
+
+    String getStr();
+
+    long getLong();
+
+    double getDouble();
+
+    boolean getBool();
+
+    String getJson();
+
+    String valueToString();
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java

@@ -0,0 +1,46 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data.dp;
+
+import lombok.Getter;
+import org.thingsboard.server.common.data.kv.DataType;
+
+public class DoubleDataPoint extends AbstractDataPoint {
+
+    @Getter
+    private final double value;
+
+    public DoubleDataPoint(long ts, double value) {
+        super(ts);
+        this.value = value;
+    }
+
+    @Override
+    public DataType getType() {
+        return DataType.DOUBLE;
+    }
+
+    @Override
+    public double getDouble() {
+        return value;
+    }
+
+    @Override
+    public String valueToString() {
+        return Double.toString(value);
+    }
+
+}

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java

@@ -0,0 +1,47 @@
+/**
+ * Copyright © 2016-2024 ThingsBoard, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.edqs.data.dp;
+
+import lombok.Getter;
+import org.thingsboard.server.common.data.kv.DataType;
+import org.thingsboard.server.edqs.repo.TbStringPool;
+
+public class JsonDataPoint extends AbstractDataPoint {
+
+    @Getter
+    private final String value;
+
+    public JsonDataPoint(long ts, String value) {
+        super(ts);
+        this.value = TbStringPool.intern(value);
+    }
+
+    @Override
+    public DataType getType() {
+        return DataType.JSON;
+    }
+
+    @Override
+    public String getJson() {
+        return value;
+    }
+
+    @Override
+    public String valueToString() {
+        return value;
+    }
+
+}