Browse Source

Merge branch 'master' into feature/separate-cassandra-rate-limits

pull/13428/head
dshvaika 1 year ago
parent
commit
a1b24b59f1
  1. 146
      application/src/main/data/json/edge/instructions/install/centos/instructions.md
  2. 54
      application/src/main/data/json/edge/instructions/install/docker/instructions.md
  3. 104
      application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md
  4. 2
      application/src/main/data/json/system/widget_bundles/high_performance_scada_energy_system.json
  5. 19
      application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java
  6. 7
      application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java
  7. 2
      application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java
  8. 24
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  9. 3
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  10. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java
  11. 3
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java
  12. 40
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java
  13. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  14. 6
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java
  15. 26
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java
  16. 155
      application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java
  17. 5
      application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java
  18. 2
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  19. 8
      application/src/main/resources/thingsboard.yml
  20. 82
      application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
  21. 48
      application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java
  22. 76
      application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java
  23. 7
      application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java
  24. 8
      application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java
  25. 21
      application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java
  26. 12
      application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java
  27. 9
      application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java
  28. 15
      application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java
  29. 9
      application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java
  30. 3
      common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java
  31. 17
      common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java
  32. 2
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java
  33. 6
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java
  34. 2
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java
  35. 18
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObjectKey.java
  36. 72
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java
  37. 4
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java
  38. 7
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java
  39. 4
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java
  40. 6
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java
  41. 5
      common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java
  42. 40
      common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java
  43. 8
      common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java
  44. 9
      common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java
  45. 2
      common/discovery-api/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java
  46. 74
      common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java
  47. 2
      common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java
  48. 3
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java
  49. 66
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java
  50. 21
      common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java
  51. 83
      common/edqs/src/main/java/org/thingsboard/server/edqs/util/DefaultEdqsMapper.java
  52. 30
      common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsMapper.java
  53. 15
      common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java
  54. 6
      common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java
  55. 5
      common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java
  56. 4
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java
  57. 23
      common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java
  58. 2
      common/proto/src/main/proto/queue.proto
  59. 7
      common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java
  60. 1
      common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java
  61. 5
      common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java
  62. 164
      common/queue/src/main/java/org/thingsboard/server/queue/common/PartitionedQueueResponseTemplate.java
  63. 5
      common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java
  64. 3
      common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java
  65. 4
      common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java
  66. 16
      common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java
  67. 35
      common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java
  68. 11
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java
  69. 2
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java
  70. 9
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java
  71. 2
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java
  72. 13
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java
  73. 4
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java
  74. 70
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsExecutors.java
  75. 5
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java
  76. 26
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java
  77. 35
      common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java
  78. 3
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java
  79. 4
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java
  80. 5
      common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java
  81. 2
      common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java
  82. 19
      common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java
  83. 5
      common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java
  84. 10
      dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java
  85. 28
      dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java
  86. 16
      dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java
  87. 10
      dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java
  88. 5
      dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java
  89. 15
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java
  90. 11
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java
  91. 20
      dao/src/main/resources/sql/schema-views-and-functions.sql
  92. 25
      dao/src/test/java/org/thingsboard/server/dao/service/EdgeEventServiceTest.java
  93. 1
      dao/src/test/resources/sql/psql/drop-all-tables.sql
  94. 14
      docker/compose-utils.sh
  95. 61
      docker/docker-compose.aws-sqs.yml
  96. 61
      docker/docker-compose.pubsub.yml
  97. 61
      docker/docker-compose.rabbitmq.yml
  98. 61
      docker/docker-compose.service-bus.yml
  99. 4
      docker/queue-aws-sqs.env
  100. 3
      docker/queue-pubsub.env

146
application/src/main/data/json/edge/instructions/install/centos/instructions.md

@ -4,15 +4,15 @@ Here is the list of commands, that can be used to quickly install ThingsBoard Ed
Before continue to installation execute the following commands in order to install necessary tools:
```bash
sudo yum install -y nano wget
sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
sudo yum install -y nano wget && sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
{:copy-code}
```
#### Install Java 17 (OpenJDK)
#### Step 1. Install Java 17 (OpenJDK)
ThingsBoard service is running on Java 17. Follow these instructions to install OpenJDK 17:
```bash
sudo yum install java-17-openjdk
sudo dnf install java-17-openjdk
{:copy-code}
```
@ -39,112 +39,94 @@ OpenJDK Runtime Environment (...)
OpenJDK 64-Bit Server VM (build ...)
```
#### Configure PostgreSQL
#### Step 2. Configure ThingsBoard Database
ThingsBoard Edge supports SQL and hybrid database approaches.
In this guide we will use SQL only.
For hybrid details please follow official installation instructions from the ThingsBoard documentation site.
### PostgresSql
ThingsBoard Edge uses PostgreSQL database as a local storage.
Instructions listed below will help you to install PostgreSQL.
To install PostgreSQL, follow the instructions below.
```bash
# Update your system
sudo yum update
sudo dnf update
{:copy-code}
```
**For CentOS 7:**
Install the repository RPM:
**For CentOS/RHEL 8:**
```bash
# Install the repository RPM (for CentOS 7):
sudo yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
# Install packages
sudo yum -y install epel-release yum-utils
sudo yum-config-manager --enable pgdg16
sudo yum install postgresql16-server postgresql16 postgresql16-contrib
# Initialize your PostgreSQL DB
sudo /usr/pgsql-16/bin/postgresql-16-setup initdb
sudo systemctl start postgresql-16
# Optional: Configure PostgreSQL to start on boot
sudo systemctl enable --now postgresql-16
# Install the repository RPM (For CentOS/RHEL 8):
sudo sudo dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
{:copy-code}
```
**For CentOS 8:**
**For CentOS/RHEL 9:**
```bash
# Install the repository RPM (for CentOS 8):
sudo yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
# Install packages
sudo dnf -qy module disable postgresql
sudo dnf -y install postgresql16 postgresql16-server postgresql16-contrib
# Initialize your PostgreSQL DB
sudo /usr/pgsql-16/bin/postgresql-16-setup initdb
sudo systemctl start postgresql-16
# Optional: Configure PostgreSQL to start on boot
sudo systemctl enable --now postgresql-16
# Install the repository RPM (for CentOS 9):
sudo dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm
{:copy-code}
```
Once PostgreSQL is installed you may want to create a new user or set the password for the main user.
The instructions below will help to set the password for main PostgreSQL user:
Install packages and initialize PostgreSQL. The PostgreSQL service will automatically start every time the system boots up.
```text
sudo su - postgres
psql
\password
\q
```bash
sudo dnf -qy module disable postgresql && \
sudo dnf -y install postgresql16 postgresql16-server postgresql16-contrib && \
sudo /usr/pgsql-16/bin/postgresql-16-setup initdb && \
sudo systemctl enable --now postgresql-16
{:copy-code}
```
Then, press "Ctrl+D" to return to main user console.
Once PostgreSQL is installed, it is recommended to set the password for the PostgreSQL main user.
After configuring the password, edit the pg_hba.conf to use MD5 authentication with the postgres user.
Edit pg_hba.conf file:
The following command will switch the current user to the PostgreSQL user and set the password directly in PostgreSQL.
```bash
sudo nano /var/lib/pgsql/16/data/pg_hba.conf
sudo -u postgres psql -c "\password"
{:copy-code}
```
Locate the following lines:
```text
# IPv4 local connections:
host all all 127.0.0.1/32 ident
```
Replace `ident` with `md5`:
Then, enter and confirm the password.
```text
host all all 127.0.0.1/32 md5
```
Since ThingsBoard Edge uses the PostgreSQL database for local storage, configuring MD5 authentication ensures that only authenticated users or
applications can access the database, thus protecting your data. After configuring the password,
edit the pg_hba.conf file to use MD5 hashing for authentication instead of the default method (ident) for local IPv4 connections.
Finally, you should restart the PostgreSQL service to initialize the new configuration:
To replace ident with md5, run the following command:
```bash
sudo systemctl restart postgresql-16.service
sudo sed -i 's/^host\s\+all\s\+all\s\+127\.0\.0\.1\/32\s\+ident/host all all 127.0.0.1\/32 md5/' /var/lib/pgsql/16/data/pg_hba.conf
{:copy-code}
```
Connect to the database to create ThingsBoard Edge DB:
Then run the command that will restart the PostgreSQL service to apply configuration changes, connect to the database as a postgres user,
and create the ThingsBoard Edge database (tb_edge). To connect to the PostgreSQL database, enter the PostgreSQL password.
```bash
psql -U postgres -d postgres -h 127.0.0.1 -W
sudo systemctl restart postgresql-16.service && psql -U postgres -d postgres -h 127.0.0.1 -W -c "CREATE DATABASE tb_edge;"
{:copy-code}
```
Execute create database statement:
#### Step 3. Choose Queue Service
```bash
CREATE DATABASE tb_edge;
\q
{:copy-code}
```
ThingsBoard Edge supports only Kafka or in-memory queue (since v4.0) for message storage and communication between ThingsBoard services.
How to choose the right queue implementation?
#### ThingsBoard Edge service installation
In Memory queue implementation is built-in and default. It is useful for development(PoC) environments and is not suitable for production deployments or any sort of cluster deployments.
Kafka is recommended for production deployments. This queue is used on the most of ThingsBoard production environments now.
In Memory queue is built in and enabled by default. No additional configuration is required.
#### Step 4. ThingsBoard Edge Service Installation
Download installation package:
```bash
wget https://github.com/thingsboard/thingsboard-edge/releases/download/v${TB_EDGE_TAG}/tb-edge-${TB_EDGE_TAG}.rpm
wget wget https://github.com/thingsboard/thingsboard-edge/releases/download/v${TB_EDGE_TAG}/tb-edge-${TB_EDGE_TAG}.rpm
{:copy-code}
```
@ -155,7 +137,7 @@ sudo rpm -Uvh tb-edge-${TB_EDGE_TAG}.rpm
{:copy-code}
```
#### Configure ThingsBoard Edge
#### Step 5. Configure ThingsBoard Edge
To configure ThingsBoard Edge, you can use the following command to automatically update the configuration file with specific values:
```bash
@ -169,25 +151,21 @@ EOL'
{:copy-code}
```
##### [Optional] Database Configuration
In case you changed default PostgreSQL datasource settings (**postgres**/**postgres**) please update the configuration file (**/etc/tb-edge/conf/tb-edge.conf**) with your actual values:
```bash
sudo nano /etc/tb-edge/conf/tb-edge.conf
{:copy-code}
```
Please update the following lines in your configuration file. Make sure **to replace**:
- Replace 'postgres' with your actual PostgreSQL username;
- Replace 'PUT_YOUR_POSTGRESQL_PASSWORD_HERE' with your actual PostgreSQL password.
##### Configure PostgreSQL (Optional)
If you changed PostgreSQL default datasource settings, use the following command:
```bash
sudo sh -c 'cat <<EOL >> /etc/tb-edge/conf/tb-edge.conf
export SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/tb_edge
export SPRING_DATASOURCE_USERNAME=postgres
export SPRING_DATASOURCE_PASSWORD=PUT_YOUR_POSTGRESQL_PASSWORD_HERE
export SPRING_DATASOURCE_PASSWORD=<PUT_YOUR_POSTGRESQL_PASSWORD_HERE>
EOL'
{:copy-code}
```
PUT_YOUR_POSTGRESQL_PASSWORD_HERE: Replace with your actual PostgreSQL user password.
##### [Optional] Update bind ports
If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server (cloud) is running, you'll need to update configuration parameters to avoid port collision between ThingsBoard server and ThingsBoard Edge.
@ -206,7 +184,7 @@ EOL'
Make sure that ports above (18080, 11883, 15683) are not used by any other application.
#### Run installation script
#### Step 6. Run installation Script
Once ThingsBoard Edge is installed and configured please execute the following install script:
```bash
@ -214,18 +192,18 @@ sudo /usr/share/tb-edge/bin/install/install.sh
{:copy-code}
```
#### Restart ThingsBoard Edge service
#### Step 7. Restart ThingsBoard Edge Service
```bash
sudo service tb-edge restart
{:copy-code}
```
#### Open ThingsBoard Edge UI
#### Step 8. Open ThingsBoard Edge UI
Once started, you will be able to open **ThingsBoard Edge UI** using the following link http://localhost:8080.
###### NOTE: Edge HTTP bind port update
Use next **ThingsBoard Edge UI** link **http://localhost:18080** if you updated HTTP 8080 bind port to **18080**.
If the Edge HTTP bind port was changed to 18080 during Edge installation, access the ThingsBoard Edge instance at http://localhost:18080.

54
application/src/main/data/json/edge/instructions/install/docker/instructions.md

@ -4,9 +4,31 @@ Here is the list of commands, that can be used to quickly install ThingsBoard Ed
Install <a href="https://docs.docker.com/engine/install/" target="_blank"> Docker CE</a> and <a href="https://docs.docker.com/compose/install/" target="_blank"> Docker Compose</a>.
#### Running ThingsBoard Edge as docker service
#### Step 1. Running ThingsBoard Edge
Create docker compose file for ThingsBoard Edge service:
Here you can find ThingsBoard Edge docker image:
<a href="https://hub.docker.com/r/thingsboard/tb-edge" target="_blank"> thingsboard/tb-edge</a>
#### Step 2. Choose Queue and/or Database Services
ThingsBoard Edge supports only Kafka or in-memory queue (since v4.0) for message storage and communication between ThingsBoard services.
ThingsBoard Edge supports SQL and hybrid database approaches.
In this guide we will use SQL only.
For hybrid details please follow official installation instructions from the ThingsBoard documentation site.
How to choose the right queue implementation?
In Memory queue implementation is built-in and default. It is useful for development(PoC) environments and is not suitable for production deployments or any sort of cluster deployments.
Kafka is recommended for production deployments. This queue is used on the most of ThingsBoard production environments now.
Hybrid implementation combines PostgreSQL and Cassandra databases with Kafka queue service. It is recommended if you plan to manage 1M+ devices in production or handle high data ingestion rate (more than 5000 msg/sec).
Create a docker compose file for the ThingsBoard Edge service:
##### In Memory
```bash
nano docker-compose.yml
@ -35,7 +57,6 @@ services:
volumes:
- tb-edge-data:/data
- tb-edge-logs:/var/log/tb-edge
${EXTRA_HOSTS}
postgres:
restart: always
image: "postgres:16"
@ -58,24 +79,20 @@ volumes:
```
##### [Optional] Update bind ports
If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server (cloud) is running, you'll need to update docker compose port mapping to avoid port collision between ThingsBoard server and ThingsBoard Edge.
If ThingsBoard Edge is set to run on the same machine where the ThingsBoard server is operating, you need to update port configuration to prevent port collision between the ThingsBoard server and ThingsBoard Edge.
Please update next lines of `docker-compose.yml` file:
Ensure that the ports 18080, 11883, 15683-15688 are not used by any other application.
```text
ports:
- "18080:8080"
- "11883:1883"
- "15683-15688:5683-5688/udp"
Then, update the port configuration in the docker-compose.yml file:
```bash
sed -i ‘s/8080:8080/18080:8080/; s/1883:1883/11883:1883/; s/5683-5688:5683-5688\/udp/15683-15688:5683-5688\/udp/’ docker-compose.yml
{:copy-code}
```
Make sure that ports above (18080, 11883, 15683-15688) are not used by any other application.
#### Start ThingsBoard Edge
Set the terminal in the directory which contains the `docker-compose.yml` file and execute the following commands to up this docker compose directly:
Set the terminal in the directory which contains the docker-compose.yml file and execute the following commands to up this docker compose directly:
```bash
docker compose up -d
docker compose logs -f mytbedge
docker compose up -d && docker compose logs -f mytbedge
{:copy-code}
```
@ -90,11 +107,12 @@ docker-compose up -d
docker-compose logs -f mytbedge
```
#### Open ThingsBoard Edge UI
#### Step 3. Open ThingsBoard Edge UI
Once started, you will be able to open **ThingsBoard Edge UI** using the following link http://localhost:8080.
Once the Edge service is started, open the Edge UI at http://localhost:8080.
###### NOTE: Edge HTTP bind port update
Use next **ThingsBoard Edge UI** link **http://localhost:18080** if you updated HTTP 8080 bind port to **18080**.
If the Edge HTTP bind port was changed to 18080 during Edge installation, access the ThingsBoard Edge instance at http://localhost:18080.
Please use your tenant credentials from local Server instance or ThingsBoard Live Demo to log in to the ThingsBoard Edge.

104
application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md

@ -1,51 +1,50 @@
Here is the list of commands, that can be used to quickly install ThingsBoard Edge on Ubuntu Server and connect to the server.
#### Install Java 17 (OpenJDK)
ThingsBoard service is running on Java 17. Follow these instructions to install OpenJDK 17:
#### Step 1. Install Java 17 (OpenJDK)
ThingsBoard service is running on Java 17. To install OpenJDK 17, follow these instructions:
```bash
sudo apt update
sudo apt install openjdk-17-jdk
sudo apt update && sudo apt install openjdk-17-jdk
{:copy-code}
```
Please don't forget to configure your operating system to use OpenJDK 17 by default.
You can configure which version is the default using the following command:
Configure your operating system to use OpenJDK 17 by default. You can configure the default version by running the following command:
```bash
sudo update-alternatives --config java
{:copy-code}
```
You can check the installation using the following command:
To check the installed Java version on your system, use the following command:
```bash
java -version
{:copy-code}
```
Expected command output is:
The expected result is:
```text
openjdk version "17.x.xx"
openjdk version "17.x.xx"
OpenJDK Runtime Environment (...)
OpenJDK 64-Bit Server VM (build ...)
OpenJDK 64-Bit Server VM (...)
```
#### Configure PostgreSQL
ThingsBoard Edge uses PostgreSQL database as a local storage.
Instructions listed below will help you to install PostgreSQL.
#### Step 2. Configure ThingsBoard Edge Database
```bash
# install **wget** if not already installed:
sudo apt install -y wget
ThingsBoard Edge supports SQL and hybrid database approaches.
In this guide we will use SQL only.
For hybrid details please follow official installation instructions from the ThingsBoard documentation site.
# import the repository signing key:
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
### Configure PostgreSQL
ThingsBoard Edge uses PostgreSQL database as a local storage.
# add repository contents to your system:
RELEASE=$(lsb_release -cs)
echo "deb http://apt.postgresql.org/pub/repos/apt/ ${RELEASE}"-pgdg main | sudo tee /etc/apt/sources.list.d/pgdg.list
To install the PostgreSQL database, run these commands:
```bash
# Automated repository configuration:
sudo apt install -y postgresql-common
sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh
# install and launch the postgresql service:
sudo apt update
@ -54,25 +53,35 @@ sudo service postgresql start
{:copy-code}
```
Once PostgreSQL is installed you may want to create a new user or set the password for the main user.
The instructions below will help to set the password for main PostgreSQL user:
Once PostgreSQL is installed, it is recommended to set the password for the PostgreSQL main user.
```text
sudo su - postgres
psql
\password
\q
The following command will switch the current user to the PostgreSQL user and set the password directly in PostgreSQL.
```bash
sudo -u postgres psql -c "\password"
{:copy-code}
```
Then, press “Ctrl+D” to return to main user console and connect to the database to create ThingsBoard Edge DB:
Then, enter and confirm the password.
```text
psql -U postgres -d postgres -h 127.0.0.1 -W
CREATE DATABASE tb_edge;
\q
Finally, create a new PostgreSQL database named tb_edge by running the following command:
```bash
echo "CREATE DATABASE tb_edge;" | psql -U postgres -d postgres -h 127.0.0.1 -W
{:copy-code}
```
#### Thingsboard Edge service installation
#### Step 3. Choose Queue Service
ThingsBoard Edge supports only Kafka or in-memory queue (since v4.0) for message storage and communication between ThingsBoard services. Choose the appropriate queue implementation based on your specific business needs:
In Memory: The built-in and default queue implementation. It is useful for development or proof-of-concept (PoC) environments, but is not recommended for production or any type of clustered deployments due to limited scalability.
Kafka: Recommended for production deployments. This queue is used in the most of ThingsBoard production environments now.
In Memory queue is built in and enabled by default. No additional configuration is required.
#### Step 4. ThingsBoard Edge Service Installation
Download installation package:
```bash
@ -87,7 +96,7 @@ sudo dpkg -i tb-edge-${TB_EDGE_TAG}.deb
{:copy-code}
```
#### Configure ThingsBoard Edge
#### Step 5. Configure ThingsBoard Edge
To configure ThingsBoard Edge, you can use the following command to automatically update the configuration file with specific values:
```bash
@ -101,25 +110,20 @@ EOL'
{:copy-code}
```
##### [Optional] Database Configuration
In case you changed default PostgreSQL datasource settings (**postgres**/**postgres**) please update the configuration file (**/etc/tb-edge/conf/tb-edge.conf**) with your actual values:
```bash
sudo nano /etc/tb-edge/conf/tb-edge.conf
{:copy-code}
```
Please update the following lines in your configuration file. Make sure **to replace**:
- Replace 'postgres' with your actual PostgreSQL username;
- Replace 'PUT_YOUR_POSTGRESQL_PASSWORD_HERE' with your actual PostgreSQL password.
##### [Optional] Configure PostgreSQL
If you changed PostgreSQL default datasource settings, use the following command:
```bash
sudo sh -c 'cat <<EOL >> /etc/tb-edge/conf/tb-edge.conf
export SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/tb_edge
export SPRING_DATASOURCE_USERNAME=postgres
export SPRING_DATASOURCE_PASSWORD=PUT_YOUR_POSTGRESQL_PASSWORD_HERE
export SPRING_DATASOURCE_PASSWORD=<PUT_YOUR_POSTGRESQL_PASSWORD_HERE>
EOL'
{:copy-code}
```
PUT_YOUR_POSTGRESQL_PASSWORD_HERE: Replace with your actual PostgreSQL user password.
##### [Optional] Update bind ports
If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server (cloud) is running, you'll need to update configuration parameters to avoid port collision between ThingsBoard server and ThingsBoard Edge.
@ -138,7 +142,7 @@ EOL'
Make sure that ports above (18080, 11883, 15683) are not used by any other application.
#### Run installation script
#### Step 6. Run installation Script
Once ThingsBoard Edge is installed and configured please execute the following install script:
@ -147,14 +151,14 @@ sudo /usr/share/tb-edge/bin/install/install.sh
{:copy-code}
```
#### Restart ThingsBoard Edge service
#### Step 7. Restart ThingsBoard Edge Service
```bash
sudo service tb-edge restart
{:copy-code}
```
#### Open ThingsBoard Edge UI
#### Step 8. Open ThingsBoard Edge UI
Once started, you will be able to open **ThingsBoard Edge UI** using the following link http://localhost:8080.

2
application/src/main/data/json/system/widget_bundles/high_performance_scada_energy_system.json

@ -15,7 +15,7 @@
"hp_wind_turbine_cluster",
"hp_fuel_generator",
"hp_industrial_fuel_generator",
"hp_circuit_breaker2",
"hp_circuit_breaker",
"hp_horizontal_circuit_breaker",
"hp_voltage_relay",
"hp_3_phase_voltage_relay",

19
application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java

@ -36,6 +36,8 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.script.api.tbel.TbelCfArg;
import org.thingsboard.script.api.tbel.TbelCfCtx;
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg;
import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal;
import org.thingsboard.script.api.tbel.TbelCfTsRollingArg;
import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
@ -240,9 +242,8 @@ public class CalculatedFieldController extends BaseController {
ctxAndArgNames.toArray(String[]::new)
);
Object[] args = new Object[ctxAndArgNames.size()];
args[0] = new TbelCfCtx(arguments);
args[0] = new TbelCfCtx(arguments, getLastUpdateTimestamp(arguments));
for (int i = 1; i < ctxAndArgNames.size(); i++) {
var arg = arguments.get(ctxAndArgNames.get(i));
if (arg instanceof TbelCfSingleValueArg svArg) {
@ -265,6 +266,20 @@ public class CalculatedFieldController extends BaseController {
return result;
}
private long getLastUpdateTimestamp(Map<String, TbelCfArg> arguments) {
long lastUpdateTimestamp = -1;
for (TbelCfArg entry : arguments.values()) {
if (entry instanceof TbelCfSingleValueArg singleValueArg) {
long ts = singleValueArg.getTs();
lastUpdateTimestamp = Math.max(lastUpdateTimestamp, ts);
} else if (entry instanceof TbelCfTsRollingArg tsRollingArg) {
long maxTs = tsRollingArg.getValues().stream().mapToLong(TbelCfTsDoubleVal::getTs).max().orElse(-1);
lastUpdateTimestamp = Math.max(lastUpdateTimestamp, maxTs);
}
}
return lastUpdateTimestamp == -1 ? System.currentTimeMillis() : lastUpdateTimestamp;
}
private <E extends HasId<I> & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException {
List<EntityId> referencedEntityIds = calculatedFieldConfig.getReferencedEntities();
for (EntityId referencedEntityId : referencedEntityIds) {

7
application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java

@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.edqs.EdqsState;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
@ -149,9 +150,9 @@ public class EntityQueryController extends BaseController {
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@GetMapping("/edqs/enabled")
public boolean isEdqsApiEnabled() {
return edqsApiService.isEnabled();
@GetMapping("/edqs/state")
public EdqsState getEdqsState() {
return edqsService.getState();
}
}

2
application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java

@ -72,7 +72,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF
@Override
public void restore(QueueKey queueKey, Set<TopicPartitionInfo> partitions) {
stateService.update(queueKey, partitions);
stateService.update(queueKey, partitions, null);
}
@Override

24
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -35,13 +35,15 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
protected Map<String, ArgumentEntry> arguments;
protected boolean sizeExceedsLimit;
protected long lastUpdateTimestamp = -1;
public BaseCalculatedFieldState(List<String> requiredArguments) {
this.requiredArguments = requiredArguments;
this.arguments = new HashMap<>();
}
public BaseCalculatedFieldState() {
this(new ArrayList<>(), new HashMap<>(), false);
this(new ArrayList<>(), new HashMap<>(), false, -1);
}
@Override
@ -59,14 +61,21 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
checkArgumentSize(key, newEntry, ctx);
ArgumentEntry existingEntry = arguments.get(key);
boolean entryUpdated;
if (existingEntry == null || newEntry.isForceResetPrevious()) {
validateNewEntry(newEntry);
arguments.put(key, newEntry);
stateUpdated = true;
entryUpdated = true;
} else {
stateUpdated = existingEntry.updateEntry(newEntry);
entryUpdated = existingEntry.updateEntry(newEntry);
}
if (entryUpdated) {
stateUpdated = true;
updateLastUpdateTimestamp(newEntry);
}
}
return stateUpdated;
@ -100,4 +109,13 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
protected abstract void validateNewEntry(ArgumentEntry newEntry);
private void updateLastUpdateTimestamp(ArgumentEntry entry) {
if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) {
this.lastUpdateTimestamp = singleValueArgumentEntry.getTs();
} else if (entry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) {
Map.Entry<Long, Double> lastEntry = tsRollingArgumentEntry.getTsRecords().lastEntry();
this.lastUpdateTimestamp = (lastEntry != null) ? lastEntry.getKey() : System.currentTimeMillis();
}
}
}

3
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
@ -61,6 +62,7 @@ public class CalculatedFieldCtx {
private final List<String> argNames;
private Output output;
private String expression;
private boolean preserveMsgTs;
private TbelInvokeService tbelInvokeService;
private CalculatedFieldScriptEngine calculatedFieldScriptEngine;
private ThreadLocal<Expression> customExpression;
@ -94,6 +96,7 @@ public class CalculatedFieldCtx {
this.argNames = new ArrayList<>(arguments.keySet());
this.output = configuration.getOutput();
this.expression = configuration.getExpression();
this.preserveMsgTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) configuration).isPreserveMsgTs();
this.tbelInvokeService = tbelInvokeService;
this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg);

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java

@ -42,6 +42,8 @@ public interface CalculatedFieldState {
Map<String, ArgumentEntry> getArguments();
long getLastUpdateTimestamp();
void setRequiredArguments(List<String> requiredArguments);
boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues);

3
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java

@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.service.cf.CalculatedFieldResult;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -67,7 +66,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState {
args.add(arg);
}
}
args.set(0, new TbelCfCtx(arguments));
args.set(0, new TbelCfCtx(arguments, getLastUpdateTimestamp()));
ListenableFuture<JsonNode> resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray());
Output output = ctx.getOutput();
return Futures.transform(resultFuture,

40
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.service.cf.ctx.state;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.Data;
@ -65,19 +67,35 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState {
double expressionResult = expr.evaluate();
Output output = ctx.getOutput();
Object result;
Integer decimals = output.getDecimalsByDefault();
if (decimals != null) {
if (decimals.equals(0)) {
result = TbUtils.toInt(expressionResult);
} else {
result = TbUtils.toFixed(expressionResult, decimals);
}
} else {
result = expressionResult;
Object result = formatResult(expressionResult, output.getDecimalsByDefault());
JsonNode outputResult = createResultJson(ctx.isPreserveMsgTs(), output.getName(), result);
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), outputResult));
}
private Object formatResult(double expressionResult, Integer decimals) {
if (decimals == null) {
return expressionResult;
}
if (decimals.equals(0)) {
return TbUtils.toInt(expressionResult);
}
return TbUtils.toFixed(expressionResult, decimals);
}
private JsonNode createResultJson(boolean preserveMsgTs, String outputName, Object result) {
ObjectNode valuesNode = JacksonUtil.newObjectNode();
valuesNode.set(outputName, JacksonUtil.valueToTree(result));
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result))));
long lastTimestamp = getLastUpdateTimestamp();
if (preserveMsgTs && lastTimestamp != -1) {
ObjectNode resultNode = JacksonUtil.newObjectNode();
resultNode.put("ts", lastTimestamp);
resultNode.set("values", valuesNode);
return resultNode;
} else {
return valuesNode;
}
}
}

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -712,7 +712,7 @@ public abstract class EdgeGrpcSession implements Closeable {
private long findStartSeqIdFromOldestEventIfAny() {
long startSeqId = 0L;
try {
TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null);
TimePageLink pageLink = new TimePageLink(1, 0, null, null, null, null);
PageData<EdgeEvent> edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink);
if (!edgeEvents.getData().isEmpty()) {
startSeqId = edgeEvents.getData().get(0).getSeqId() - 1;

6
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java

@ -26,9 +26,13 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.edge.EdgeEventService;
import java.util.concurrent.TimeUnit;
@AllArgsConstructor
@Slf4j
public class GeneralEdgeEventFetcher implements EdgeEventFetcher {
// Subtract from queueStartTs to ensure no data is lost due to potential misordering of edge events by created_time.
private static final long MISORDERING_COMPENSATION_MILLIS = TimeUnit.SECONDS.toMillis(60);
private final Long queueStartTs;
private Long seqIdStart;
@ -44,7 +48,7 @@ public class GeneralEdgeEventFetcher implements EdgeEventFetcher {
0,
null,
null,
queueStartTs,
queueStartTs > 0 ? queueStartTs - MISORDERING_COMPENSATION_MILLIS : 0,
System.currentTimeMillis());
}

26
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java

@ -22,7 +22,6 @@ import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
@ -51,11 +50,6 @@ public class DefaultEdqsApiService implements EdqsApiService {
private final EdqsClientQueueFactory queueFactory;
private TbQueueRequestTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> requestTemplate;
@Value("${queue.edqs.api.auto_enable:true}")
private boolean autoEnable;
private Boolean apiEnabled = null;
@PostConstruct
private void init() {
requestTemplate = queueFactory.createEdqsRequestTemplate();
@ -85,31 +79,11 @@ public class DefaultEdqsApiService implements EdqsApiService {
}, MoreExecutors.directExecutor());
}
@Override
public boolean isEnabled() {
return Boolean.TRUE.equals(apiEnabled);
}
@Override
public void setEnabled(boolean enabled) {
if (enabled) {
log.info("Enabling EDQS API");
} else {
log.info("Disabling EDQS API");
}
apiEnabled = enabled;
}
@Override
public boolean isSupported() {
return true;
}
@Override
public boolean isAutoEnable() {
return autoEnable;
}
@PreDestroy
private void stop() {
requestTemplate.stop();

155
application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java

@ -20,11 +20,13 @@ import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@ -36,6 +38,9 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsEventType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsState;
import org.thingsboard.server.common.data.edqs.EdqsState.EdqsApiMode;
import org.thingsboard.server.common.data.edqs.EdqsState.EdqsSyncStatus;
import org.thingsboard.server.common.data.edqs.EdqsSyncRequest;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
@ -51,20 +56,25 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.edqs.processor.EdqsProducer;
import org.thingsboard.server.edqs.state.EdqsPartitionService;
import org.thingsboard.server.edqs.util.EdqsConverter;
import org.thingsboard.server.edqs.util.DefaultEdqsMapper;
import org.thingsboard.server.edqs.util.EdqsMapper;
import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.discovery.DiscoveryService;
import org.thingsboard.server.queue.discovery.HashPartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.environment.DistributedLock;
import org.thingsboard.server.queue.environment.DistributedLockService;
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Service
@ -74,31 +84,42 @@ import java.util.concurrent.TimeUnit;
public class DefaultEdqsService implements EdqsService {
private final EdqsClientQueueFactory queueFactory;
private final EdqsConverter edqsConverter;
private final EdqsMapper edqsMapper;
private final EdqsSyncService edqsSyncService;
private final EdqsApiService edqsApiService;
private final DistributedLockService distributedLockService;
private final AttributesService attributesService;
private final EdqsPartitionService edqsPartitionService;
private final TopicService topicService;
private final TbServiceInfoProvider serviceInfoProvider;
private final DiscoveryService discoveryService;
@Autowired @Lazy
private TbClusterService clusterService;
@Autowired @Lazy
private HashPartitionService hashPartitionService;
@Value("${queue.edqs.api.auto_enable:true}")
private boolean autoEnableApi;
@Value("${queue.edqs.readiness_check_interval:60000}")
private int edqsReadinessCheckInterval;
private EdqsProducer eventsProducer;
private ExecutorService executor;
private ScheduledExecutorService scheduler;
private DistributedLock syncLock;
@Getter
private EdqsState state;
@PostConstruct
private void init() {
executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass());
scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-check");
eventsProducer = EdqsProducer.builder()
.producer(queueFactory.createEdqsEventsProducer())
.partitionService(edqsPartitionService)
.build();
syncLock = distributedLockService.getLock("edqs_sync");
state = new EdqsState();
}
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
@ -106,6 +127,26 @@ public class DefaultEdqsService implements EdqsService {
if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) {
return;
}
if (edqsApiService.isSupported()) {
scheduler.scheduleWithFixedDelay(() -> {
if (!hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) {
return;
}
List<ServiceInfo> servers = new ArrayList<>(discoveryService.getOtherServers());
servers.add(serviceInfoProvider.getServiceInfo());
List<ServiceInfo> readyEdqsServers = servers.stream()
.filter(serviceInfo -> serviceInfo.getServiceTypesList().contains(ServiceType.EDQS.name()))
.filter(ServiceInfo::getReady)
.toList();
boolean changed = state.setEdqsReady(!readyEdqsServers.isEmpty());
if (changed) {
broadcastEdqsReady(state.getEdqsReady());
}
}, 0, edqsReadinessCheckInterval, TimeUnit.MILLISECONDS);
}
executor.submit(() -> {
try {
EdqsSyncState syncState = getSyncState();
@ -115,9 +156,9 @@ public class DefaultEdqsService implements EdqsService {
.syncRequest(new EdqsSyncRequest())
.build());
}
} else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) {
} else {
// only if topic/RocksDB is not empty and sync is finished
edqsApiService.setEnabled(true);
onSyncStatusUpdate(EdqsSyncStatus.FINISHED);
}
} catch (Throwable e) {
log.error("Failed to start EDQS service", e);
@ -131,7 +172,10 @@ public class DefaultEdqsService implements EdqsService {
if (request.getSyncRequest() != null) {
saveSyncState(EdqsSyncStatus.REQUESTED);
}
broadcast(request.toInternalMsg());
broadcast(ToCoreEdqsMsg.builder()
.syncRequest(request.getSyncRequest())
.apiEnabled(request.getApiEnabled())
.build());
}
@Override
@ -140,7 +184,18 @@ public class DefaultEdqsService implements EdqsService {
log.info("Processing system msg {}", msg);
try {
if (msg.getApiEnabled() != null) {
edqsApiService.setEnabled(msg.getApiEnabled());
if (msg.getApiEnabled()) {
state.setApiMode(EdqsApiMode.ENABLED);
} else {
state.setApiMode(EdqsApiMode.DISABLED);
}
log.info("New state: {}", state);
}
if (msg.getEdqsReady() != null) {
onEdqsReady(msg.getEdqsReady());
}
if (msg.getSyncStatus() != null) {
onSyncStatusUpdate(msg.getSyncStatus());
}
if (msg.getSyncRequest() != null) {
@ -154,23 +209,16 @@ public class DefaultEdqsService implements EdqsService {
return;
}
}
saveSyncState(EdqsSyncStatus.STARTED);
edqsSyncService.sync();
saveSyncState(EdqsSyncStatus.FINISHED);
if (edqsApiService.isSupported())
if (edqsApiService.isAutoEnable()) {
log.info("EDQS sync is finished, auto-enabling API");
broadcast(ToCoreEdqsMsg.builder()
.apiEnabled(Boolean.TRUE)
.build());
} else {
log.info("EDQS sync is finished, but leaving API disabled");
}
saveSyncState(EdqsSyncStatus.FINISHED);
broadcastSyncStatusUpdate(EdqsSyncStatus.FINISHED);
} catch (Exception e) {
log.error("Failed to complete sync", e);
saveSyncState(EdqsSyncStatus.FAILED);
broadcastSyncStatusUpdate(EdqsSyncStatus.FAILED);
} finally {
syncLock.unlock();
}
@ -181,6 +229,60 @@ public class DefaultEdqsService implements EdqsService {
});
}
private void broadcastEdqsReady(boolean ready) {
broadcast(ToCoreEdqsMsg.builder()
.edqsReady(ready)
.build());
}
private void onEdqsReady(boolean ready) {
state.setEdqsReady(ready);
checkState();
}
private void broadcastSyncStatusUpdate(EdqsSyncStatus status) {
broadcast(ToCoreEdqsMsg.builder()
.syncStatus(status)
.build());
}
private void onSyncStatusUpdate(EdqsSyncStatus status) {
state.setSyncStatus(status);
checkState();
}
private void checkState() {
if (!edqsApiService.isSupported()) {
log.info("New state: {}. EDQS API not supported", state);
return;
}
if (state.isApiReady()) {
if (autoEnableApi) {
if (state.getApiMode() == null || state.getApiMode() == EdqsApiMode.AUTO_DISABLED) {
state.setApiMode(EdqsApiMode.AUTO_ENABLED);
log.info("New state: {}. Auto-enabled EDQS API", state);
} else {
log.info("New state: {}. API mode left as is", state);
}
} else {
log.info("New state: {}. API auto-enabling is disabled", state);
}
} else {
if (state.isApiEnabled()) {
state.setApiMode(EdqsApiMode.AUTO_DISABLED);
log.info("New state: {}. Disabled EDQS API", state);
} else {
log.info("New state: {}. API left disabled", state);
}
}
}
@Override
public boolean isApiEnabled() {
return state.isApiEnabled();
}
@Override
public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {
EntityType entityType = entityId.getEntityType();
@ -189,7 +291,7 @@ public class DefaultEdqsService implements EdqsService {
log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType);
return;
}
onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity));
onUpdate(tenantId, objectType, DefaultEdqsMapper.toEntity(entityType, entity));
}
@Override
@ -216,12 +318,11 @@ public class DefaultEdqsService implements EdqsService {
protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) {
executor.submit(() -> {
try {
String key = object.key();
String key = object.stringKey();
Long version = object.version();
EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder()
.setKey(key)
.setObjectType(objectType.name())
.setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object)))
.setData(ByteString.copyFrom(edqsMapper.serialize(object)))
.setEventType(eventType.name());
if (version != null) {
eventMsg.setVersion(version);
@ -278,6 +379,7 @@ public class DefaultEdqsService implements EdqsService {
@PreDestroy
private void stop() {
executor.shutdown();
scheduler.shutdownNow();
eventsProducer.stop();
}
@ -288,11 +390,4 @@ public class DefaultEdqsService implements EdqsService {
private EdqsSyncStatus status;
}
private enum EdqsSyncStatus {
REQUESTED,
STARTED,
FINISHED,
FAILED
}
}

5
application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.edqs;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
@ -32,11 +33,11 @@ public class KafkaEdqsSyncService extends EdqsSyncService {
private final boolean syncNeeded;
public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings, EdqsConfig edqsConfig) {
public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings, TopicService topicService, EdqsConfig edqsConfig) {
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap());
this.syncNeeded = kafkaAdmin.areAllTopicsEmpty(IntStream.range(0, edqsConfig.getPartitions())
.mapToObj(partition -> TopicPartitionInfo.builder()
.topic(edqsConfig.getEventsTopic())
.topic(topicService.buildTopicName(edqsConfig.getEventsTopic()))
.partition(partition)
.build().getFullTopicName())
.collect(Collectors.toSet()));

2
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -251,7 +251,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
mainConsumer.update(event.getCorePartitions());
usageStatsConsumer.subscribe(event.getCorePartitions()
.stream()
.map(tpi -> tpi.newByTopic(usageStatsConsumer.getConsumer().getTopic()))
.map(tpi -> tpi.withTopic(usageStatsConsumer.getConsumer().getTopic()))
.collect(Collectors.toSet()));
}

8
application/src/main/resources/thingsboard.yml

@ -668,7 +668,7 @@ cache:
# Spring data parameters
spring.data.redis.repositories.enabled: false # Disable this because it is not required.
# Redis configuration parameters
# Redis/Valkey configuration parameters
redis:
# standalone or cluster or sentinel
connection:
@ -1764,6 +1764,8 @@ queue:
supported: "${TB_EDQS_API_SUPPORTED:false}"
# Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished
auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}"
# Interval in milliseconds to check for ready EDQS servers
readiness_check_interval: "${TB_EDQS_READINESS_CHECK_INTERVAL_MS:60000}"
# Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices)
mode: "${TB_EDQS_MODE:local}"
local:
@ -1787,6 +1789,10 @@ queue:
max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}"
# Maximum timeout for requests to EDQS
max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}"
# Thread pool size for EDQS requests executor
request_executor_size: "${TB_EDQS_REQUEST_EXECUTOR_SIZE:50}"
# Time to live for EDQS versions cache in minutes. Must be bigger than the time taken for the sync process.
versions_cache_ttl: "${TB_EDQS_VERSIONS_CACHE_TTL_MINUTES:60}"
# Strings longer than this threshold will be compressed
string_compression_length_threshold: "${TB_EDQS_STRING_COMPRESSION_LENGTH_THRESHOLD:512}"
stats:

82
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.debug.DebugSettings;
import org.thingsboard.server.common.data.id.AssetProfileId;
@ -462,6 +463,87 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
});
}
@Test
public void testSimpleCalculatedFieldWhenPreserveMsgTsIsTrue() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
long ts = System.currentTimeMillis() - 300000L;
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts)));
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(testDevice.getId());
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName("C to F");
calculatedField.setDebugSettings(DebugSettings.all());
calculatedField.setConfigurationVersion(1);
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
Argument argument = new Argument();
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("T", argument));
config.setExpression("(T * 9/5) + 32");
Output output = new Output();
output.setName("fahrenheitTemp");
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
config.setPreserveMsgTs(true);
calculatedField.setConfiguration(config);
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0");
});
}
@Test
public void testScriptCalculatedFieldWhenUsedMsgTsInScript() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
long ts = System.currentTimeMillis() - 300000L;
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts)));
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(testDevice.getId());
calculatedField.setType(CalculatedFieldType.SCRIPT);
calculatedField.setName("C to F");
calculatedField.setDebugSettings(DebugSettings.all());
calculatedField.setConfigurationVersion(1);
ScriptCalculatedFieldConfiguration config = new ScriptCalculatedFieldConfiguration();
Argument argument = new Argument();
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("T", argument));
config.setExpression("return {\"ts\": ctx.msgTs, \"values\": {\"fahrenheitTemp\": (T * 1.8) + 32}};");
Output output = new Output();
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
calculatedField.setConfiguration(config);
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("ts").asText()).isEqualTo(Long.toString(ts));
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0");
});
}
private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception {
return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class);
}

48
application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java

@ -60,6 +60,7 @@ import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
@ -68,6 +69,7 @@ import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.security.model.JwtSettings;
import org.thingsboard.server.dao.edge.EdgeDao;
import org.thingsboard.server.dao.exception.DataValidationException;
@ -107,6 +109,7 @@ import java.util.concurrent.TimeUnit;
import static org.hamcrest.Matchers.containsString;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
import static org.thingsboard.server.edge.AbstractEdgeTest.CONNECT_MESSAGE_COUNT;
@TestPropertySource(properties = {
"edges.enabled=true",
@ -138,6 +141,7 @@ public class EdgeControllerTest extends AbstractControllerTest {
public EdgeDao edgeDao(EdgeDao edgeDao) {
return Mockito.mock(EdgeDao.class, AdditionalAnswers.delegatesTo(edgeDao));
}
}
@Before
@ -886,6 +890,8 @@ public class EdgeControllerTest extends AbstractControllerTest {
Device savedDevice = doPost("/api/device", device, Device.class);
// create public customer
//1 message
// Customer
doPost("/api/customer/public/device/" + savedDevice.getId().getId(), Device.class);
doDelete("/api/customer/device/" + savedDevice.getId().getId(), Device.class);
@ -897,13 +903,16 @@ public class EdgeControllerTest extends AbstractControllerTest {
+ "/asset/" + savedAsset.getId().getId().toString(), Asset.class);
EdgeImitator edgeImitator = new EdgeImitator(EDGE_HOST, EDGE_PORT, edge.getRoutingKey(), edge.getSecret());
edgeImitator.ignoreType(UserCredentialsUpdateMsg.class);
edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class);
edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class);
edgeImitator.expectMessageAmount(27);
// 17 connect message
// + 1 Customer
// + 5 fetchers messages (DeviceProfile, Device, DeviceCredentials, AssetProfile, Asset) in sync process
// + 5 queue messages the same
edgeImitator.expectMessageAmount(CONNECT_MESSAGE_COUNT + 11);
edgeImitator.connect();
waitForMessages(edgeImitator);
edgeImitator.waitForMessages();
verifyFetchersMsgs(edgeImitator, savedDevice);
// verify queue msgs
@ -914,9 +923,12 @@ public class EdgeControllerTest extends AbstractControllerTest {
Assert.assertTrue(popAssetMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Test Sync Edge Asset 1"));
printQueueMsgsIfNotEmpty(edgeImitator);
edgeImitator.expectMessageAmount(21);
// 17 connect messages
// + 1 Customer
// + 5 fetchers messages (DeviceProfile, Device, DeviceCredentials, AssetProfile, Asset) in sync process
edgeImitator.expectMessageAmount(CONNECT_MESSAGE_COUNT + 6);
doPost("/api/edge/sync/" + edge.getId()).andExpect(status().isOk());
waitForMessages(edgeImitator);
edgeImitator.waitForMessages();
verifyFetchersMsgs(edgeImitator, savedDevice);
printQueueMsgsIfNotEmpty(edgeImitator);
@ -987,17 +999,6 @@ public class EdgeControllerTest extends AbstractControllerTest {
});
}
private void waitForMessages(EdgeImitator edgeImitator) throws Exception {
boolean success = edgeImitator.waitForMessages();
if (!success) {
List<AbstractMessage> downlinkMsgs = edgeImitator.getDownlinkMsgs();
for (AbstractMessage downlinkMsg : downlinkMsgs) {
log.error("{}\n{}", downlinkMsg.getClass(), downlinkMsg);
}
Assert.fail("Await for messages was not successful!");
}
}
private void verifyFetchersMsgs(EdgeImitator edgeImitator, Device savedDevice) {
Assert.assertTrue(popQueueMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Main"));
Assert.assertTrue(popRuleChainMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Edge Root Rule Chain"));
@ -1011,6 +1012,7 @@ public class EdgeControllerTest extends AbstractControllerTest {
Assert.assertTrue(popAssetProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default"));
Assert.assertTrue(popDeviceProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default"));
Assert.assertTrue(popAssetProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default"));
Assert.assertTrue(popUserCredentialsMsg(edgeImitator.getDownlinkMsgs(), currentUserId));
Assert.assertTrue(popUserMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, TENANT_ADMIN_EMAIL, Authority.TENANT_ADMIN));
Assert.assertTrue(popCustomerMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Public"));
Assert.assertTrue(popDeviceProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default"));
@ -1156,6 +1158,20 @@ public class EdgeControllerTest extends AbstractControllerTest {
return false;
}
private boolean popUserCredentialsMsg(List<AbstractMessage> messages, UserId userId) {
for (AbstractMessage message : messages) {
if (message instanceof UserCredentialsUpdateMsg userCredentialsUpdateMsg) {
UserCredentials userCredentials = JacksonUtil.fromString(userCredentialsUpdateMsg.getEntity(), UserCredentials.class, true);
Assert.assertNotNull(userCredentials);
if (userId.equals(userCredentials.getUserId())) {
messages.remove(message);
return true;
}
}
}
return false;
}
private boolean popUserMsg(List<AbstractMessage> messages, UpdateMsgType msgType, String email, Authority authority) {
for (AbstractMessage message : messages) {
if (message instanceof UserUpdateMsg userUpdateMsg) {

76
application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java

@ -15,21 +15,27 @@
*/
package org.thingsboard.server.controller;
import org.assertj.core.api.ThrowingConsumer;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.edqs.EdqsState;
import org.thingsboard.server.common.data.edqs.EdqsState.EdqsApiMode;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edqs.state.EdqsStateService;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
import org.thingsboard.server.queue.discovery.DiscoveryService;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
@DaoSqlTest
@ -39,22 +45,23 @@ import static org.awaitility.Awaitility.await;
"queue.edqs.sync.enabled=true",
"queue.edqs.api.supported=true",
"queue.edqs.api.auto_enable=true",
"queue.edqs.mode=local"
"queue.edqs.mode=local",
"queue.edqs.readiness_check_interval=500"
})
public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest {
@Autowired
private EdqsApiService edqsApiService;
private EdqsService edqsService;
@Autowired
private EdqsStateService edqsStateService;
private DiscoveryService discoveryService;
@MockBean // so that we don't do backup for tests
private EdqsRocksDb edqsRocksDb;
@Before
public void before() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled() && edqsStateService.isReady());
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.getState().isApiEnabled());
}
@Override
@ -69,4 +76,61 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest {
result -> result == expectedResult);
}
@Test
public void testEdqsState() {
assertThat(edqsService.getState().getApiMode()).isEqualTo(EdqsApiMode.AUTO_ENABLED);
// notifying EDQS is not ready: API should be auto-disabled
discoveryService.setReady(false);
verifyState(state -> {
assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.AUTO_DISABLED);
assertThat(state.getEdqsReady()).isFalse();
assertThat(state.isApiEnabled()).isFalse();
});
// manually disabling API
edqsService.processSystemRequest(ToCoreEdqsRequest.builder()
.apiEnabled(false)
.build());
verifyState(state -> {
assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.DISABLED);
assertThat(state.getEdqsReady()).isFalse();
assertThat(state.isApiEnabled()).isFalse();
});
// notifying EDQS is ready: API should not be enabled automatically because manually disabled previously
discoveryService.setReady(true);
verifyState(state -> {
assertThat(state.getEdqsReady()).isTrue();
assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.DISABLED);
assertThat(state.isApiEnabled()).isFalse();
});
// manually enabling API
edqsService.processSystemRequest(ToCoreEdqsRequest.builder()
.apiEnabled(true)
.build());
verifyState(state -> {
assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.ENABLED);
assertThat(state.getEdqsReady()).isTrue();
assertThat(state.isApiEnabled()).isTrue();
});
// notifying EDQS is not ready: API should be auto-disabled
discoveryService.setReady(false);
verifyState(state -> {
assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.AUTO_DISABLED);
assertThat(state.getEdqsReady()).isFalse();
assertThat(state.isApiEnabled()).isFalse();
});
discoveryService.setReady(true);
}
private void verifyState(ThrowingConsumer<EdqsState> assertion) {
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
assertThat(edqsService.getState()).satisfies(assertion);
});
}
}

7
application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@ -22,6 +22,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.web.servlet.ResultActions;
import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
import org.thingsboard.common.util.JacksonUtil;
@ -78,6 +79,10 @@ import static org.awaitility.Awaitility.await;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@DaoSqlTest
@TestPropertySource(properties = {
"queue.edqs.sync.enabled=true", // only enabling sync
"queue.edqs.api.supported=false",
})
public class EntityQueryControllerTest extends AbstractControllerTest {
private static final String CUSTOMER_USER_EMAIL = "entityQueryCustomer@thingsboard.org";
@ -803,7 +808,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
//assign dashboard
doPost("/api/customer/" + savedCustomer.getId().getId().toString()
+ "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
+ "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
// check entity data query by customer
User customerUser = new User();

8
application/src/test/java/org/thingsboard/server/controller/TrendzControllerTest.java

@ -27,6 +27,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
public class TrendzControllerTest extends AbstractControllerTest {
private final String trendzUrl = "https://some.domain.com:18888/also_necessary_prefix";
private final String apiKey = "$2a$10$iDjfqYmnrw9gkdw4XhgzFOU.R/pVz3OKgXOdpbR2LuXaKatGcGLiG";
@Before
public void setUp() throws Exception {
@ -35,6 +36,7 @@ public class TrendzControllerTest extends AbstractControllerTest {
TrendzSettings trendzSettings = new TrendzSettings();
trendzSettings.setEnabled(true);
trendzSettings.setBaseUrl(trendzUrl);
trendzSettings.setApiKey(apiKey);
doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk());
}
@ -48,9 +50,12 @@ public class TrendzControllerTest extends AbstractControllerTest {
assertThat(trendzSettings).isNotNull();
assertThat(trendzSettings.isEnabled()).isTrue();
assertThat(trendzSettings.getBaseUrl()).isEqualTo(trendzUrl);
trendzSettings.setApiKey(apiKey);
String updatedUrl = "https://some.domain.com:18888/tenant_trendz";
String updatedApiKey = "$2a$10$aRR0bHa8rtzP5jRcE72vp.hRFsGQz4MGIs62oogLbfOCFK3.RIESG";
trendzSettings.setBaseUrl(updatedUrl);
trendzSettings.setApiKey(updatedApiKey);
doPost("/api/trendz/settings", trendzSettings).andExpect(status().isOk());
@ -65,6 +70,7 @@ public class TrendzControllerTest extends AbstractControllerTest {
TrendzSettings newTrendzSettings = new TrendzSettings();
newTrendzSettings.setEnabled(true);
newTrendzSettings.setBaseUrl("https://some.domain.com:18888/customer_trendz");
newTrendzSettings.setApiKey("some_api_key");
doPost("/api/trendz/settings", newTrendzSettings).andExpect(status().isForbidden());
@ -72,6 +78,6 @@ public class TrendzControllerTest extends AbstractControllerTest {
assertThat(fetchedTrendzSettings).isNotNull();
assertThat(fetchedTrendzSettings.isEnabled()).isTrue();
assertThat(fetchedTrendzSettings.getBaseUrl()).isEqualTo(trendzUrl);
assertThat(fetchedTrendzSettings.getApiKey()).isEqualTo(apiKey);
}
}

21
application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java

@ -115,7 +115,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
})
@Slf4j
abstract public class AbstractEdgeTest extends AbstractControllerTest {
public static final Integer CONNECT_MESSAGE_COUNT = 17;
public static final Integer INSTALLATION_MESSAGE_COUNT = 8;
public static final Integer SYNC_MESSAGE_COUNT = CONNECT_MESSAGE_COUNT + INSTALLATION_MESSAGE_COUNT;
private static final String THERMOSTAT_DEVICE_PROFILE_NAME = "Thermostat";
protected DeviceProfile thermostatDeviceProfile;
@ -136,11 +138,12 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
doPost("/api/admin/jwtSettings", settings).andExpect(status().isOk());
loginTenantAdmin();
//8 installation messages
installation();
edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret());
edgeImitator.expectMessageAmount(25);
// 17 connect messages + 8 installation messages
edgeImitator.expectMessageAmount(SYNC_MESSAGE_COUNT);
edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class);
edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class);
edgeImitator.connect();
@ -164,22 +167,32 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest {
thermostatDeviceProfile = this.createDeviceProfile(THERMOSTAT_DEVICE_PROFILE_NAME,
createMqttDeviceProfileTransportConfiguration(new JsonTransportPayloadConfiguration(), false));
extendDeviceProfileData(thermostatDeviceProfile);
//2 messages DeviceProfile
thermostatDeviceProfile = doPost("/api/deviceProfile", thermostatDeviceProfile, DeviceProfile.class);
Device savedDevice = saveDevice("Edge Device 1", THERMOSTAT_DEVICE_PROFILE_NAME);
// create public customer
//1 message
// Customer
doPost("/api/customer/public/device/" + savedDevice.getId().getId(), Device.class);
doDelete("/api/customer/device/" + savedDevice.getId().getId(), Device.class);
Asset savedAsset = saveAsset("Edge Asset 1");
Asset savedAsset = saveAsset("Edge Asset 1");
updateRootRuleChainMetadata();
edge = doPost("/api/edge", constructEdge("Test Edge", "test"), Edge.class);
//3 messages
// Device
// DeviceProfile
// DeviceCredentials
doPost("/api/edge/" + edge.getUuidId()
+ "/device/" + savedDevice.getUuidId(), Device.class);
//2 messages
// Asset
// AssetProfile
doPost("/api/edge/" + edge.getUuidId()
+ "/asset/" + savedAsset.getUuidId(), Asset.class);

12
application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java

@ -128,15 +128,19 @@ public class DeviceProfileEdgeTest extends AbstractEdgeTest {
@Test
public void testDeleteDeviceProfilesWhenEdgeIsOffline() throws Exception {
//2 message RuleChain and RuleChainMetadata
RuleChainId thermostatsRuleChainId = createEdgeRuleChainAndAssignToEdge("Thermostats Rule Chain");
// create device profile
DeviceProfile deviceProfile = this.createDeviceProfile("ONE_MORE_DEVICE_PROFILE", null);
deviceProfile.setDefaultEdgeRuleChainId(thermostatsRuleChainId);
extendDeviceProfileData(deviceProfile);
//1 message DeviceProfile
edgeImitator.expectMessageAmount(1);
deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class);
Assert.assertTrue(edgeImitator.waitForMessages());
AbstractMessage latestMessage = edgeImitator.getLatestMessage();
Assert.assertTrue(latestMessage instanceof DeviceProfileUpdateMsg);
DeviceProfileUpdateMsg deviceProfileUpdateMsg = (DeviceProfileUpdateMsg) latestMessage;
@ -150,9 +154,11 @@ public class DeviceProfileEdgeTest extends AbstractEdgeTest {
doDelete("/api/deviceProfile/" + deviceProfile.getUuidId())
.andExpect(status().isOk());
edgeImitator.connect();
// 27 sync message
// + 1 delete message
edgeImitator.expectMessageAmount(28);
// 25 sync message
// + 2 RuleChain and RuleChainMetadata
// + 1 delete DeviceProfile
edgeImitator.expectMessageAmount(SYNC_MESSAGE_COUNT + 3);
Assert.assertTrue(edgeImitator.waitForMessages());
latestMessage = edgeImitator.getLatestMessage();

9
application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java

@ -78,11 +78,14 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest {
TenantProfileQueueConfiguration mainQueueConfiguration = createQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC);
TenantProfileQueueConfiguration isolatedQueueConfiguration = createQueueConfig("IsolatedHighPriority", "tb_rule_engine.isolated_hp");
edgeTenantProfile.getProfileData().setQueueConfiguration(List.of(mainQueueConfiguration, isolatedQueueConfiguration));
// + 1 TenantProfile
// + 1 Queue main
// + 1 Queue isolated
edgeImitator.expectMessageAmount(3);
edgeTenantProfile = doPost("/api/tenantProfile", edgeTenantProfile, TenantProfile.class);
Assert.assertTrue(edgeImitator.waitForMessages());
Optional<TenantProfileUpdateMsg> tenantProfileUpdateMsgOpt = edgeImitator.findMessageByType(TenantProfileUpdateMsg.class);
Optional<TenantProfileUpdateMsg> tenantProfileUpdateMsgOpt = edgeImitator.findMessageByType(TenantProfileUpdateMsg.class);
Assert.assertTrue(tenantProfileUpdateMsgOpt.isPresent());
TenantProfileUpdateMsg tenantProfileUpdateMsg = tenantProfileUpdateMsgOpt.get();
TenantProfile tenantProfile = JacksonUtil.fromString(tenantProfileUpdateMsg.getEntity(), TenantProfile.class, true);
@ -96,7 +99,9 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest {
loginTenantAdmin();
edgeImitator.expectMessageAmount(21);
// 25 sync message
// +1 isolated Queue
edgeImitator.expectMessageAmount(SYNC_MESSAGE_COUNT + 1);
doPost("/api/edge/sync/" + edge.getId());
assertThat(edgeImitator.waitForMessages()).as("await for messages after edge sync rest api call").isTrue();

15
application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java

@ -24,6 +24,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.junit.Assert;
import org.thingsboard.edge.rpc.EdgeGrpcClient;
import org.thingsboard.edge.rpc.EdgeRpcClient;
import org.thingsboard.server.controller.AbstractWebTest;
@ -386,7 +387,19 @@ public class EdgeImitator {
}
public boolean waitForMessages() throws InterruptedException {
return waitForMessages(AbstractWebTest.TIMEOUT);
boolean success = waitForMessages(AbstractWebTest.TIMEOUT);
if (!success) {
List<AbstractMessage> downlinkMsgs = getDownlinkMsgs();
for (AbstractMessage downlinkMsg : downlinkMsgs) {
log.error("{}\n{}", downlinkMsg.getClass(), downlinkMsg);
}
log.error("message count: {}", downlinkMsgs.size());
Assert.fail("Await for messages was not successful!");
}
return true;
}
public boolean waitForMessages(int timeoutInSeconds) throws InterruptedException {

9
application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java

@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.common.data.relation.EntitySearchDirection;
import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.edqs.util.EdqsRocksDb;
@ -53,19 +53,20 @@ import static org.awaitility.Awaitility.await;
"queue.edqs.sync.enabled=true",
"queue.edqs.api.supported=true",
"queue.edqs.api.auto_enable=true",
"queue.edqs.mode=local"
"queue.edqs.mode=local",
"queue.edqs.readiness_check_interval=1000"
})
public class EdqsEntityServiceTest extends EntityServiceTest {
@Autowired
private EdqsApiService edqsApiService;
private EdqsService edqsService;
@MockBean
private EdqsRocksDb edqsRocksDb;
@Before
public void beforeEach() {
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled());
await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled());
}
// sql implementation has a bug with data duplication, edqs implementation returns correct value

3
common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java

@ -19,11 +19,10 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
public interface TbQueueProducer<T extends TbQueueMsg> {
void init();
String getDefaultTopic();
void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback);
void stop();
}

17
common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java

@ -18,6 +18,7 @@ package org.thingsboard.server.common.data;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
@ -76,6 +77,15 @@ public enum EntityType {
public static final List<String> NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream()
.map(EntityType::getNormalName).toList();
private static final EntityType[] BY_PROTO;
static {
BY_PROTO = new EntityType[Arrays.stream(values()).mapToInt(EntityType::getProtoNumber).max().orElse(0) + 1];
for (EntityType entityType : values()) {
BY_PROTO[entityType.getProtoNumber()] = entityType;
}
}
EntityType(int protoNumber) {
this.protoNumber = protoNumber;
this.tableName = name().toLowerCase();
@ -98,4 +108,11 @@ public enum EntityType {
return false;
}
public static EntityType forProtoNumber(int protoNumber) {
if (protoNumber < 0 || protoNumber >= BY_PROTO.length) {
throw new IllegalArgumentException("Invalid EntityType proto number " + protoNumber);
}
return BY_PROTO[protoNumber];
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java

@ -23,6 +23,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType;
@EqualsAndHashCode(callSuper = true)
public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration {
private boolean preserveMsgTs;
@Override
public CalculatedFieldType getType() {
return CalculatedFieldType.SIMPLE;

6
common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java

@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import java.util.UUID;
@Data
@AllArgsConstructor
@NoArgsConstructor
@ -58,7 +60,7 @@ public class AttributeKv implements EdqsObject {
}
@Override
public String key() {
public String stringKey() {
return "a_" + entityId + "_" + scope + "_" + key;
}
@ -72,4 +74,6 @@ public class AttributeKv implements EdqsObject {
return ObjectType.ATTRIBUTE_KV;
}
public record Key(UUID entityId, AttributeScope scope, int key) implements EdqsObjectKey {}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java

@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.ObjectType;
public interface EdqsObject {
@JsonIgnore
String key();
String stringKey();
@JsonIgnore
Long version();

18
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObjectKey.java

@ -0,0 +1,18 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
public interface EdqsObjectKey {}

72
common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java

@ -0,0 +1,72 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.edqs;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.commons.lang3.BooleanUtils;
@Getter
@NoArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class EdqsState {
private Boolean edqsReady;
@Setter
private EdqsSyncStatus syncStatus;
@Setter
private EdqsApiMode apiMode;
public boolean setEdqsReady(boolean ready) {
boolean changed = BooleanUtils.toBooleanDefaultIfNull(this.edqsReady, false) != ready;
this.edqsReady = ready;
return changed;
}
public boolean isApiReady() {
return edqsReady && syncStatus == EdqsSyncStatus.FINISHED;
}
public boolean isApiEnabled() {
return apiMode != null && (apiMode == EdqsApiMode.ENABLED || apiMode == EdqsApiMode.AUTO_ENABLED);
}
@Override
public String toString() {
return '[' +
"EDQS ready: " + edqsReady +
", sync status: " + syncStatus +
", API mode: " + apiMode +
']';
}
public enum EdqsSyncStatus {
REQUESTED,
STARTED,
FINISHED,
FAILED
}
public enum EdqsApiMode {
ENABLED,
AUTO_ENABLED,
DISABLED,
AUTO_DISABLED
}
}

4
common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java

@ -51,7 +51,7 @@ public class Entity implements EdqsObject {
}
@Override
public String key() {
public String stringKey() {
return "e_" + fields.getId().toString();
}
@ -65,4 +65,6 @@ public class Entity implements EdqsObject {
return ObjectType.fromEntityType(type);
}
public record Key(UUID id) implements EdqsObjectKey {}
}

7
common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java

@ -24,6 +24,8 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.UUID;
@Data
@AllArgsConstructor
@NoArgsConstructor
@ -53,7 +55,8 @@ public class LatestTsKv implements EdqsObject {
this.version = version != null ? version : 0L;
}
public String key() {
@Override
public String stringKey() {
return "l_" + entityId + "_" + key;
}
@ -67,4 +70,6 @@ public class LatestTsKv implements EdqsObject {
return ObjectType.LATEST_TS_KV;
}
public record Key(UUID entityId, int key) implements EdqsObjectKey {}
}

4
common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java

@ -19,6 +19,7 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.edqs.EdqsState.EdqsSyncStatus;
@Data
@AllArgsConstructor
@ -29,4 +30,7 @@ public class ToCoreEdqsMsg {
private EdqsSyncRequest syncRequest;
private Boolean apiEnabled;
private EdqsSyncStatus syncStatus;
private Boolean edqsReady;
}

6
common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.common.data.edqs;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -30,9 +29,4 @@ public class ToCoreEdqsRequest {
private EdqsSyncRequest syncRequest;
private Boolean apiEnabled;
@JsonIgnore
public ToCoreEdqsMsg toInternalMsg() {
return new ToCoreEdqsMsg(syncRequest, apiEnabled);
}
}

5
common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java

@ -29,9 +29,7 @@ public interface EntityFields {
Logger log = LoggerFactory.getLogger(EntityFields.class);
default UUID getId() {
return null;
}
UUID getId();
default UUID getTenantId() {
return null;
@ -147,6 +145,7 @@ public interface EntityFields {
default String getAsString(String key) {
return switch (key) {
case "id" -> getId().toString();
case "createdTime" -> Long.toString(getCreatedTime());
case "title" -> getName();
case "type" -> getType();

40
common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentLifecycleEvent.java

@ -15,12 +15,42 @@
*/
package org.thingsboard.server.common.data.plugin;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import java.io.Serializable;
import java.util.Arrays;
/**
* @author Andrew Shvayka
*/
@RequiredArgsConstructor
public enum ComponentLifecycleEvent implements Serializable {
// In sync with ComponentLifecycleEvent proto
CREATED, STARTED, ACTIVATED, SUSPENDED, UPDATED, STOPPED, DELETED, FAILED, DEACTIVATED
CREATED(0),
STARTED(1),
ACTIVATED(2),
SUSPENDED(3),
UPDATED(4),
STOPPED(5),
DELETED(6),
FAILED(7),
DEACTIVATED(8);
@Getter
private final int protoNumber; // corresponds to ComponentLifecycleEvent proto
private static final ComponentLifecycleEvent[] BY_PROTO;
static {
BY_PROTO = new ComponentLifecycleEvent[Arrays.stream(values()).mapToInt(ComponentLifecycleEvent::getProtoNumber).max().orElse(0) + 1];
for (ComponentLifecycleEvent event : values()) {
BY_PROTO[event.getProtoNumber()] = event;
}
}
public static ComponentLifecycleEvent forProtoNumber(int protoNumber) {
if (protoNumber < 0 || protoNumber >= BY_PROTO.length) {
throw new IllegalArgumentException("Invalid ComponentLifecycleEvent proto number " + protoNumber);
}
return BY_PROTO[protoNumber];
}
}

8
common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java

@ -27,10 +27,12 @@ import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo;
import org.thingsboard.server.common.data.HasVersion;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsObjectKey;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.validation.Length;
import java.io.Serializable;
import java.util.UUID;
@Slf4j
@Schema
@ -119,8 +121,8 @@ public class EntityRelation implements HasVersion, Serializable, EdqsObject {
BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes);
}
@JsonIgnore
public String key() {
@Override
public String stringKey() {
return "r_" + from + "_" + to + "_" + typeGroup + "_" + type;
}
@ -134,4 +136,6 @@ public class EntityRelation implements HasVersion, Serializable, EdqsObject {
return ObjectType.RELATION;
}
public record Key(UUID from, UUID to, RelationTypeGroup typeGroup, String type) implements EdqsObjectKey {}
}

9
common/data/src/main/java/org/thingsboard/server/common/data/trendz/TrendzSettings.java

@ -15,12 +15,19 @@
*/
package org.thingsboard.server.common.data.trendz;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
@Data
public class TrendzSettings {
@NoArgsConstructor
@AllArgsConstructor
public class TrendzSettings implements Serializable {
private boolean enabled;
private String baseUrl;
private String apiKey;
}

2
common/discovery-api/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java

@ -37,4 +37,6 @@ public interface TbServiceInfoProvider {
Set<UUID> getAssignedTenantProfiles();
boolean setReady(boolean ready);
}

74
common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java

@ -18,7 +18,6 @@ package org.thingsboard.server.edqs.processor;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
@ -29,7 +28,6 @@ import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ExceptionUtil;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsEvent;
@ -47,30 +45,31 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.edqs.repo.EdqsRepository;
import org.thingsboard.server.edqs.state.EdqsPartitionService;
import org.thingsboard.server.edqs.state.EdqsStateService;
import org.thingsboard.server.edqs.util.EdqsConverter;
import org.thingsboard.server.edqs.util.EdqsMapper;
import org.thingsboard.server.edqs.util.VersionsStore;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg;
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueHandler;
import org.thingsboard.server.queue.TbQueueResponseTemplate;
import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import org.thingsboard.server.queue.discovery.DiscoveryService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.edqs.EdqsComponent;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy;
import org.thingsboard.server.queue.edqs.EdqsExecutors;
import org.thingsboard.server.queue.edqs.EdqsQueueFactory;
import org.thingsboard.server.queue.util.AfterStartUp;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -84,23 +83,20 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop
public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> {
private final EdqsQueueFactory queueFactory;
private final EdqsConverter converter;
private final EdqsMapper mapper;
private final EdqsRepository repository;
private final EdqsConfig config;
private final EdqsExecutors edqsExecutors;
private final EdqsPartitionService partitionService;
private final DiscoveryService discoveryService;
private final TopicService topicService;
private final ConfigurableApplicationContext applicationContext;
private final EdqsStateService stateService;
private PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer;
private TbQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> responseTemplate;
private ExecutorService consumersExecutor;
private ExecutorService taskExecutor;
private ScheduledExecutorService scheduler;
private PartitionedQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> responseTemplate;
private ListeningExecutorService requestExecutor;
private final VersionsStore versionsStore = new VersionsStore();
private VersionsStore versionsStore;
private final AtomicInteger counter = new AtomicInteger();
@Getter
@ -108,22 +104,21 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
@PostConstruct
private void init() {
consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer"));
taskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor");
scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler");
requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests"));
errorHandler = error -> {
if (error instanceof OutOfMemoryError) {
log.error("OOM detected, shutting down");
repository.clear();
discoveryService.setReady(false);
Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-shutdown"))
.execute(applicationContext::close);
}
};
requestExecutor = edqsExecutors.getRequestExecutor();
versionsStore = new VersionsStore(config.getVersionsCacheTtl());
eventConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToEdqsMsg>>create()
.queueKey(new QueueKey(ServiceType.EDQS, config.getEventsTopic()))
.topic(config.getEventsTopic())
.topic(topicService.buildTopicName(config.getEventsTopic()))
.pollInterval(config.getPollInterval())
.msgPackProcessor((msgs, consumer, config) -> {
for (TbProtoQueueMsg<ToEdqsMsg> queueMsg : msgs) {
@ -141,19 +136,14 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
})
.consumerCreator((config, tpi) -> queueFactory.createEdqsEventsConsumer())
.queueAdmin(queueFactory.getEdqsQueueAdmin())
.consumerExecutor(consumersExecutor)
.taskExecutor(taskExecutor)
.scheduler(scheduler)
.consumerExecutor(edqsExecutors.getConsumersExecutor())
.taskExecutor(edqsExecutors.getConsumerTaskExecutor())
.scheduler(edqsExecutors.getScheduler())
.uncaughtErrorHandler(errorHandler)
.build();
stateService.init(eventConsumer);
responseTemplate = queueFactory.createEdqsResponseTemplate(this);
responseTemplate = queueFactory.createEdqsResponseTemplate();
}
@AfterStartUp(order = 1)
public void start() {
responseTemplate.launch(this);
stateService.init(eventConsumer, List.of(responseTemplate.getRequestConsumer()));
}
@EventListener
@ -163,10 +153,8 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
}
try {
Set<TopicPartitionInfo> newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS));
stateService.process(withTopic(newPartitions, config.getStateTopic()));
// eventsConsumer's partitions are updated by stateService
responseTemplate.subscribe(withTopic(newPartitions, config.getRequestsTopic())); // TODO: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template
stateService.process(withTopic(newPartitions, topicService.buildTopicName(config.getStateTopic())));
// partitions for event and request consumers are updated by stateService
Set<TopicPartitionInfo> oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS));
if (CollectionsUtil.isNotEmpty(oldPartitions)) {
@ -241,22 +229,20 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
TenantId tenantId = getTenantId(edqsMsg);
ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType());
EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType());
String key = eventMsg.getKey();
Long version = eventMsg.hasVersion() ? eventMsg.getVersion() : null;
EdqsObject object = mapper.deserialize(objectType, eventMsg.getData().toByteArray(), false);
if (version != null) {
if (!versionsStore.isNew(key, version)) {
if (!versionsStore.isNew(mapper.getKey(object), version)) {
return;
}
} else if (!ObjectType.unversionedTypes.contains(objectType)) {
log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key);
log.warn("[{}] {} doesn't have version: {}", tenantId, objectType, object);
}
if (backup) {
stateService.save(tenantId, objectType, key, eventType, edqsMsg);
stateService.save(tenantId, objectType, object.stringKey(), eventType, edqsMsg);
}
EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray());
log.debug("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version);
int count = counter.incrementAndGet();
if (count % 100000 == 0) {
log.info("Processed {} events", count);
@ -268,6 +254,7 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
.eventType(eventType)
.object(object)
.build();
log.debug("Processing event: {}", event);
repository.processEvent(event);
}
}
@ -290,11 +277,6 @@ public class EdqsProcessor implements TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>,
eventConsumer.awaitStop();
responseTemplate.stop();
stateService.stop();
consumersExecutor.shutdownNow();
taskExecutor.shutdownNow();
scheduler.shutdownNow();
requestExecutor.shutdownNow();
}
}

2
common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java

@ -35,7 +35,7 @@ public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor<Entit
@Override
protected boolean matches(EntityData ed) {
return ed.getFields() != null && (pattern == null || pattern.matcher(ed.getFields().getName()).matches());
return super.matches(ed) && (pattern == null || pattern.matcher(ed.getFields().getName()).matches());
}
}

3
common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java

@ -23,11 +23,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import java.util.List;
import java.util.Set;
public interface EdqsStateService {
void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer);
void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer, List<PartitionedQueueConsumerManager<?>> otherConsumers);
void process(Set<TopicPartitionInfo> partitions);

66
common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java

@ -22,11 +22,13 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsEventType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.edqs.processor.EdqsProcessor;
import org.thingsboard.server.edqs.processor.EdqsProducer;
import org.thingsboard.server.edqs.util.EdqsMapper;
import org.thingsboard.server.edqs.util.VersionsStore;
import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
@ -34,15 +36,18 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import org.thingsboard.server.queue.common.consumer.QueueConsumerManager;
import org.thingsboard.server.queue.common.state.KafkaQueueStateService;
import org.thingsboard.server.queue.common.state.QueueStateService;
import org.thingsboard.server.queue.discovery.DiscoveryService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.edqs.EdqsConfig;
import org.thingsboard.server.queue.edqs.EdqsExecutors;
import org.thingsboard.server.queue.edqs.KafkaEdqsComponent;
import org.thingsboard.server.queue.edqs.KafkaEdqsQueueFactory;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
@ -59,26 +64,32 @@ public class KafkaEdqsStateService implements EdqsStateService {
private final EdqsConfig config;
private final EdqsPartitionService partitionService;
private final KafkaEdqsQueueFactory queueFactory;
private final DiscoveryService discoveryService;
private final EdqsExecutors edqsExecutors;
private final EdqsMapper mapper;
private final TopicService topicService;
@Autowired
@Lazy
private EdqsProcessor edqsProcessor;
private PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> stateConsumer;
private QueueStateService<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<ToEdqsMsg>> queueStateService;
private KafkaQueueStateService<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<ToEdqsMsg>> queueStateService;
private QueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventsToBackupConsumer;
private EdqsProducer stateProducer;
private VersionsStore versionsStore;
private final VersionsStore versionsStore = new VersionsStore();
private final AtomicInteger stateReadCount = new AtomicInteger();
private final AtomicInteger eventsReadCount = new AtomicInteger();
private Boolean ready;
private boolean ready = false;
@Override
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer) {
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer, List<PartitionedQueueConsumerManager<?>> otherConsumers) {
versionsStore = new VersionsStore(config.getVersionsCacheTtl());
TbKafkaAdmin queueAdmin = queueFactory.getEdqsQueueAdmin();
stateConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToEdqsMsg>>create()
.queueKey(new QueueKey(ServiceType.EDQS, config.getStateTopic()))
.topic(config.getStateTopic())
.topic(topicService.buildTopicName(config.getStateTopic()))
.pollInterval(config.getPollInterval())
.msgPackProcessor((msgs, consumer, config) -> {
for (TbProtoQueueMsg<ToEdqsMsg> queueMsg : msgs) {
@ -96,9 +107,9 @@ public class KafkaEdqsStateService implements EdqsStateService {
})
.consumerCreator((config, tpi) -> queueFactory.createEdqsStateConsumer())
.queueAdmin(queueAdmin)
.consumerExecutor(eventConsumer.getConsumerExecutor())
.taskExecutor(eventConsumer.getTaskExecutor())
.scheduler(eventConsumer.getScheduler())
.consumerExecutor(edqsExecutors.getConsumersExecutor())
.taskExecutor(edqsExecutors.getConsumerTaskExecutor())
.scheduler(edqsExecutors.getScheduler())
.uncaughtErrorHandler(edqsProcessor.getErrorHandler())
.build();
@ -117,22 +128,25 @@ public class KafkaEdqsStateService implements EdqsStateService {
if (msg.hasEventMsg()) {
EdqsEventMsg eventMsg = msg.getEventMsg();
String key = eventMsg.getKey();
int count = eventsReadCount.incrementAndGet();
if (count % 100000 == 0) {
log.info("[events-to-backup] Processed {} msgs", count);
}
ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType());
EdqsObject object = mapper.deserialize(objectType, eventMsg.getData().toByteArray(), true);
if (eventMsg.hasVersion()) {
if (!versionsStore.isNew(key, eventMsg.getVersion())) {
if (!versionsStore.isNew(mapper.getKey(object), eventMsg.getVersion())) {
continue;
}
}
TenantId tenantId = getTenantId(msg);
ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType());
EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType());
String key = object.stringKey();
log.trace("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key);
stateProducer.send(tenantId, objectType, key, msg);
stateProducer.send(tenantId, objectType, object.stringKey(), msg);
int count = eventsReadCount.incrementAndGet();
if (count % 100000 == 0) {
log.info("[events-to-backup] Processed {} msgs", count);
}
}
} catch (Throwable t) {
log.error("Failed to process message: {}", queueMsg, t);
@ -141,7 +155,7 @@ public class KafkaEdqsStateService implements EdqsStateService {
consumer.commit();
})
.consumerCreator(() -> eventsToBackupKafkaConsumer)
.consumerExecutor(eventConsumer.getConsumerExecutor())
.consumerExecutor(edqsExecutors.getConsumersExecutor())
.threadPrefix("edqs-events-to-backup")
.build();
@ -153,6 +167,7 @@ public class KafkaEdqsStateService implements EdqsStateService {
queueStateService = KafkaQueueStateService.<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<ToEdqsMsg>>builder()
.eventConsumer(eventConsumer)
.stateConsumer(stateConsumer)
.otherConsumers(otherConsumers)
.eventsStartOffsetsProvider(() -> {
// taking start offsets for events topics from the events-to-backup consumer group,
// since eventConsumer doesn't use consumer group management and thus offset tracking
@ -176,14 +191,17 @@ public class KafkaEdqsStateService implements EdqsStateService {
if (queueStateService.getPartitions().isEmpty()) {
Set<TopicPartitionInfo> allPartitions = IntStream.range(0, config.getPartitions())
.mapToObj(partition -> TopicPartitionInfo.builder()
.topic(config.getEventsTopic())
.topic(topicService.buildTopicName(config.getEventsTopic()))
.partition(partition)
.build())
.collect(Collectors.toSet());
eventsToBackupConsumer.subscribe(allPartitions);
eventsToBackupConsumer.launch();
}
queueStateService.update(new QueueKey(ServiceType.EDQS), partitions);
queueStateService.update(new QueueKey(ServiceType.EDQS), partitions, () -> {
ready = true;
discoveryService.setReady(true);
});
}
@Override
@ -193,13 +211,7 @@ public class KafkaEdqsStateService implements EdqsStateService {
@Override
public boolean isReady() {
if (ready == null) {
Set<TopicPartitionInfo> partitionsInProgress = queueStateService.getPartitionsInProgress();
if (partitionsInProgress != null && partitionsInProgress.isEmpty()) {
ready = true; // once true - always true, not to change readiness status on each repartitioning
}
}
return ready != null && ready;
return ready;
}
private TenantId getTenantId(ToEdqsMsg edqsMsg) {

21
common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java

@ -29,8 +29,10 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import org.thingsboard.server.queue.discovery.DiscoveryService;
import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent;
import java.util.List;
import java.util.Set;
import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic;
@ -42,20 +44,24 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop
public class LocalEdqsStateService implements EdqsStateService {
private final EdqsRocksDb db;
private final DiscoveryService discoveryService;
@Autowired @Lazy
private EdqsProcessor processor;
private PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer;
private Set<TopicPartitionInfo> partitions;
private List<PartitionedQueueConsumerManager<?>> otherConsumers;
private boolean ready = false;
@Override
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer) {
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToEdqsMsg>> eventConsumer, List<PartitionedQueueConsumerManager<?>> otherConsumers) {
this.eventConsumer = eventConsumer;
this.otherConsumers = otherConsumers;
}
@Override
public void process(Set<TopicPartitionInfo> partitions) {
if (this.partitions == null) {
if (!ready) {
db.forEach((key, value) -> {
try {
ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value);
@ -67,8 +73,13 @@ public class LocalEdqsStateService implements EdqsStateService {
});
log.info("Restore completed");
}
ready = true;
discoveryService.setReady(true);
eventConsumer.update(withTopic(partitions, eventConsumer.getTopic()));
this.partitions = partitions;
for (PartitionedQueueConsumerManager<?> consumer : otherConsumers) {
consumer.update(withTopic(partitions, consumer.getTopic()));
}
}
@Override
@ -87,7 +98,7 @@ public class LocalEdqsStateService implements EdqsStateService {
@Override
public boolean isReady() {
return partitions != null;
return ready;
}
@Override

83
common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java → common/edqs/src/main/java/org/thingsboard/server/edqs/util/DefaultEdqsMapper.java

@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.protobuf.ByteString;
import lombok.RequiredArgsConstructor;
@ -36,6 +35,7 @@ import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.AttributeKv;
import org.thingsboard.server.common.data.edqs.DataPoint;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsObjectKey;
import org.thingsboard.server.common.data.edqs.Entity;
import org.thingsboard.server.common.data.edqs.LatestTsKv;
import org.thingsboard.server.common.data.edqs.fields.FieldsUtil;
@ -52,6 +52,7 @@ import org.thingsboard.server.edqs.data.dp.DoubleDataPoint;
import org.thingsboard.server.edqs.data.dp.JsonDataPoint;
import org.thingsboard.server.edqs.data.dp.LongDataPoint;
import org.thingsboard.server.edqs.data.dp.StringDataPoint;
import org.thingsboard.server.edqs.repo.KeyDictionary;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto;
import org.xerial.snappy.Snappy;
@ -65,19 +66,29 @@ import java.util.UUID;
@Service
@RequiredArgsConstructor
@Slf4j
public class EdqsConverter {
public class DefaultEdqsMapper implements EdqsMapper {
private final EdqsStatsService edqsStatsService;
@Value("${queue.edqs.string_compression_length_threshold:512}")
private int stringCompressionLengthThreshold;
private final Map<ObjectType, Converter<? extends EdqsObject>> converters = new HashMap<>();
private final Converter<Entity> defaultConverter = new JsonConverter<>(Entity.class);
private final Map<ObjectType, Mapper<? extends EdqsObject>> mappers = new HashMap<>();
private final Mapper<Entity> defaultMapper = new JsonMapper<>(Entity.class) {
@Override
public EdqsObjectKey getKey(Entity entity) {
return new Entity.Key(entity.getFields().getId());
}
};
{
converters.put(ObjectType.RELATION, new JsonConverter<>(EntityRelation.class));
converters.put(ObjectType.ATTRIBUTE_KV, new Converter<AttributeKv>() {
mappers.put(ObjectType.RELATION, new JsonMapper<>(EntityRelation.class) {
@Override
public EdqsObjectKey getKey(EntityRelation relation) {
return new EntityRelation.Key(relation.getFrom().getId(), relation.getTo().getId(), relation.getTypeGroup(), relation.getType());
}
});
mappers.put(ObjectType.ATTRIBUTE_KV, new Mapper<AttributeKv>() {
@Override
public byte[] serialize(ObjectType type, AttributeKv attributeKv) {
var proto = TransportProtos.AttributeKvProto.newBuilder()
@ -94,12 +105,12 @@ public class EdqsConverter {
}
@Override
public AttributeKv deserialize(ObjectType type, byte[] bytes) throws Exception {
public AttributeKv deserialize(ObjectType type, byte[] bytes, boolean onlyKey) throws Exception {
TransportProtos.AttributeKvProto proto = TransportProtos.AttributeKvProto.parseFrom(bytes);
EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()),
new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
AttributeScope scope = AttributeScope.values()[proto.getScope().getNumber()];
DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null;
DataPoint dataPoint = onlyKey || !proto.hasDataPoint() ? null : fromDataPointProto(proto.getDataPoint());
return AttributeKv.builder()
.entityId(entityId)
.scope(scope)
@ -108,8 +119,13 @@ public class EdqsConverter {
.dataPoint(dataPoint)
.build();
}
@Override
public EdqsObjectKey getKey(AttributeKv attributeKv) {
return new AttributeKv.Key(attributeKv.getEntityId().getId(), attributeKv.getScope(), KeyDictionary.get(attributeKv.getKey()));
}
});
converters.put(ObjectType.LATEST_TS_KV, new Converter<LatestTsKv>() {
mappers.put(ObjectType.LATEST_TS_KV, new Mapper<LatestTsKv>() {
@Override
public byte[] serialize(ObjectType type, LatestTsKv latestTsKv) {
var proto = TransportProtos.LatestTsKvProto.newBuilder()
@ -125,11 +141,11 @@ public class EdqsConverter {
}
@Override
public LatestTsKv deserialize(ObjectType type, byte[] bytes) throws Exception {
public LatestTsKv deserialize(ObjectType type, byte[] bytes, boolean onlyKey) throws Exception {
TransportProtos.LatestTsKvProto proto = TransportProtos.LatestTsKvProto.parseFrom(bytes);
EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()),
new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null;
DataPoint dataPoint = onlyKey || !proto.hasDataPoint() ? null : fromDataPointProto(proto.getDataPoint());
return LatestTsKv.builder()
.entityId(entityId)
.key(proto.getKey())
@ -137,6 +153,11 @@ public class EdqsConverter {
.dataPoint(dataPoint)
.build();
}
@Override
public EdqsObjectKey getKey(LatestTsKv latestTsKv) {
return new LatestTsKv.Key(latestTsKv.getEntityId().getId(), KeyDictionary.get(latestTsKv.getKey()));
}
});
}
@ -223,30 +244,30 @@ public class EdqsConverter {
@SuppressWarnings("unchecked")
@SneakyThrows
public <T extends EdqsObject> byte[] serialize(ObjectType type, T value) {
Converter<T> converter = (Converter<T>) converters.get(type);
if (converter != null) {
return converter.serialize(type, value);
} else {
return defaultConverter.serialize(type, (Entity) value);
}
public <T extends EdqsObject> byte[] serialize(T value) {
ObjectType type = value.type();
Mapper<T> mapper = (Mapper<T>) mappers.getOrDefault(type, defaultMapper);
return mapper.serialize(type, value);
}
@SneakyThrows
public EdqsObject deserialize(ObjectType type, byte[] bytes) {
Converter<? extends EdqsObject> converter = converters.get(type);
if (converter != null) {
return converter.deserialize(type, bytes);
} else {
return defaultConverter.deserialize(type, bytes);
}
public EdqsObject deserialize(ObjectType type, byte[] bytes, boolean onlyKey) {
Mapper<? extends EdqsObject> mapper = mappers.getOrDefault(type, defaultMapper);
return mapper.deserialize(type, bytes, onlyKey);
}
@SuppressWarnings("unchecked")
@SneakyThrows
public <T extends EdqsObject> EdqsObjectKey getKey(T object) {
Mapper<T> mapper = (Mapper<T>) mappers.getOrDefault(object.type(), defaultMapper);
return mapper.getKey(object);
}
@RequiredArgsConstructor
private static class JsonConverter<T> implements Converter<T> {
private static abstract class JsonMapper<T> implements Mapper<T> {
private static final SimpleModule module = new SimpleModule();
private static final ObjectMapper mapper = JsonMapper.builder()
private static final ObjectMapper mapper = com.fasterxml.jackson.databind.json.JsonMapper.builder()
.visibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY)
.visibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE)
.visibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE)
@ -267,17 +288,19 @@ public class EdqsConverter {
@SneakyThrows
@Override
public T deserialize(ObjectType objectType, byte[] bytes) {
public T deserialize(ObjectType objectType, byte[] bytes, boolean onlyKey) {
return mapper.readValue(bytes, this.type);
}
}
private interface Converter<T> {
private interface Mapper<T> {
byte[] serialize(ObjectType type, T value) throws Exception;
T deserialize(ObjectType type, byte[] bytes) throws Exception;
T deserialize(ObjectType type, byte[] bytes, boolean onlyKey) throws Exception;
EdqsObjectKey getKey(T object);
}

30
common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsMapper.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.edqs.util;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsObjectKey;
public interface EdqsMapper {
<T extends EdqsObject> byte[] serialize(T value);
EdqsObject deserialize(ObjectType type, byte[] bytes, boolean onlyKey);
<T extends EdqsObject> EdqsObjectKey getKey(T object);
}

15
common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java

@ -18,6 +18,7 @@ package org.thingsboard.server.edqs.util;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.edqs.EdqsObjectKey;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -25,18 +26,22 @@ import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j
public class VersionsStore {
private final Cache<String, Long> versions = Caffeine.newBuilder()
.expireAfterWrite(24, TimeUnit.HOURS)
.build();
private final Cache<EdqsObjectKey, Long> versions;
public boolean isNew(String key, Long version) {
public VersionsStore(int ttlMinutes) {
this.versions = Caffeine.newBuilder()
.expireAfterWrite(ttlMinutes, TimeUnit.MINUTES)
.build();
}
public boolean isNew(EdqsObjectKey key, Long version) {
AtomicBoolean isNew = new AtomicBoolean(false);
versions.asMap().compute(key, (k, prevVersion) -> {
if (prevVersion == null || prevVersion <= version) {
isNew.set(true);
return version;
} else {
log.info("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion);
log.debug("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion);
return prevVersion;
}
});

6
common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java

@ -25,12 +25,6 @@ public interface EdqsApiService {
ListenableFuture<EdqsResponse> processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request);
boolean isEnabled();
void setEnabled(boolean enabled);
boolean isSupported();
boolean isAutoEnable();
}

5
common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java

@ -17,6 +17,7 @@ package org.thingsboard.server.common.msg.edqs;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsState;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.id.EntityId;
@ -36,4 +37,8 @@ public interface EdqsService {
void processSystemMsg(ToCoreEdqsMsg request);
boolean isApiEnabled();
EdqsState getState();
}

4
common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java

@ -57,10 +57,6 @@ public class TopicPartitionInfo {
this(topic, tenantId, partition, false, myPartition);
}
public TopicPartitionInfo newByTopic(String topic) {
return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition);
}
public String getTopic() {
return topic;
}

23
common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java

@ -102,7 +102,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ApiUsageRecordKeyPro
import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.UUID;
@ -114,14 +113,6 @@ import static org.thingsboard.server.common.data.DataConstants.GATEWAY_PARAMETER
@Slf4j
public class ProtoUtils {
private static final EntityType[] entityTypeByProtoNumber;
static {
int arraySize = Arrays.stream(EntityType.values()).mapToInt(EntityType::getProtoNumber).max().orElse(0);
entityTypeByProtoNumber = new EntityType[arraySize + 1];
Arrays.stream(EntityType.values()).forEach(entityType -> entityTypeByProtoNumber[entityType.getProtoNumber()] = entityType);
}
public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) {
var builder = TransportProtos.ComponentLifecycleMsgProto.newBuilder()
.setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits())
@ -129,7 +120,7 @@ public class ProtoUtils {
.setEntityType(toProto(msg.getEntityId().getEntityType()))
.setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits())
.setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits())
.setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal()));
.setEvent(toProto(msg.getEvent()));
if (msg.getProfileId() != null) {
builder.setProfileIdMSB(msg.getProfileId().getId().getMostSignificantBits());
builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits());
@ -156,7 +147,7 @@ public class ProtoUtils {
var builder = ComponentLifecycleMsg.builder()
.tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())))
.entityId(entityId)
.event(ComponentLifecycleEvent.values()[proto.getEventValue()]);
.event(fromProto(proto.getEvent()));
if (!StringUtils.isEmpty(proto.getName())) {
builder.name(proto.getName());
}
@ -175,7 +166,15 @@ public class ProtoUtils {
}
public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) {
return entityTypeByProtoNumber[entityType.getNumber()];
return EntityType.forProtoNumber(entityType.getNumber());
}
public static TransportProtos.ComponentLifecycleEvent toProto(ComponentLifecycleEvent event) {
return TransportProtos.ComponentLifecycleEvent.forNumber(event.getProtoNumber());
}
public static ComponentLifecycleEvent fromProto(TransportProtos.ComponentLifecycleEvent eventProto) {
return ComponentLifecycleEvent.forProtoNumber(eventProto.getNumber());
}
public static TransportProtos.ToEdgeSyncRequestMsgProto toProto(ToEdgeSyncRequest request) {

2
common/proto/src/main/proto/queue.proto

@ -89,6 +89,7 @@ message ServiceInfo {
SystemInfoProto systemInfo = 10;
repeated string assignedTenantProfiles = 11;
string label = 12;
bool ready = 13;
}
message SystemInfoProto {
@ -1836,7 +1837,6 @@ message FromEdqsMsg {
}
message EdqsEventMsg {
string key = 1;
string objectType = 2;
bytes data = 3;
string eventType = 4;

7
common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java

@ -114,6 +114,13 @@ class ProtoUtilsTest {
}
}
@Test
void protoComponentLifecycleEventSerialization() {
for (ComponentLifecycleEvent event : ComponentLifecycleEvent.values()) {
assertThat(ProtoUtils.fromProto(ProtoUtils.toProto(event))).isEqualTo(event);
}
}
@Test
void protoEdgeHighPrioritySerialization() {
EdgeHighPriorityMsg msg = new EdgeHighPriorityMsg(tenantId, EdgeUtils.constructEdgeEvent(tenantId, edgeId,

1
common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java

@ -91,7 +91,6 @@ public class DefaultTbQueueRequestTemplate<Request extends TbQueueMsg, Response
@Override
public void init() {
queueAdmin.createTopicIfNotExists(responseTemplate.getTopic());
requestTemplate.init();
responseTemplate.subscribe();
executor.submit(this::mainLoop);
}

5
common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java

@ -30,8 +30,6 @@ import org.thingsboard.server.queue.TbQueueResponseTemplate;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@ -44,7 +42,6 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
private final TbQueueConsumer<Request> requestTemplate;
private final TbQueueProducer<Response> responseTemplate;
private final ConcurrentMap<UUID, String> pendingRequests;
private final ExecutorService loopExecutor;
private final ScheduledExecutorService timeoutExecutor;
private final ExecutorService callbackExecutor;
@ -67,7 +64,6 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
MessagesStats stats) {
this.requestTemplate = requestTemplate;
this.responseTemplate = responseTemplate;
this.pendingRequests = new ConcurrentHashMap<>();
this.maxPendingRequests = maxPendingRequests;
this.pollInterval = pollInterval;
this.requestTimeout = requestTimeout;
@ -89,7 +85,6 @@ public class DefaultTbQueueResponseTemplate<Request extends TbQueueMsg, Response
@Override
public void launch(TbQueueHandler<Request, Response> handler) {
this.responseTemplate.init();
loopExecutor.submit(() -> {
while (!stopped) {
try {

164
common/queue/src/main/java/org/thingsboard/server/queue/common/PartitionedQueueResponseTemplate.java

@ -0,0 +1,164 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.common;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.stats.MessagesStats;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueHandler;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
@Slf4j
public class PartitionedQueueResponseTemplate<Request extends TbQueueMsg, Response extends TbQueueMsg> extends AbstractTbQueueTemplate {
@Getter
private final PartitionedQueueConsumerManager<Request> requestConsumer;
private final TbQueueProducer<Response> responseProducer;
private final TbQueueHandler<Request, Response> handler;
private final long pollInterval;
private final int maxPendingRequests;
private final long requestTimeout;
private final MessagesStats stats;
private final ScheduledExecutorService scheduler;
private final ExecutorService callbackExecutor;
private final AtomicInteger pendingRequestCount = new AtomicInteger();
@Builder
public PartitionedQueueResponseTemplate(String key,
TbQueueHandler<Request, Response> handler,
String requestsTopic,
Function<TopicPartitionInfo, TbQueueConsumer<Request>> consumerCreator,
TbQueueProducer<Response> responseProducer,
long pollInterval,
long requestTimeout,
int maxPendingRequests,
ExecutorService consumerExecutor,
ExecutorService callbackExecutor,
ExecutorService consumerTaskExecutor,
MessagesStats stats) {
this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor(key + "-queue-response-template-scheduler");
this.callbackExecutor = callbackExecutor;
this.handler = handler;
this.requestConsumer = PartitionedQueueConsumerManager.<Request>create()
.queueKey(key + "-requests")
.topic(requestsTopic)
.pollInterval(pollInterval)
.msgPackProcessor((requests, consumer, config) -> processRequests(requests, consumer))
.consumerCreator((config, tpi) -> consumerCreator.apply(tpi))
.consumerExecutor(consumerExecutor)
.scheduler(scheduler)
.taskExecutor(consumerTaskExecutor)
.build();
this.responseProducer = responseProducer;
this.pollInterval = pollInterval;
this.maxPendingRequests = maxPendingRequests;
this.requestTimeout = requestTimeout;
this.stats = stats;
}
private void processRequests(List<Request> requests, TbQueueConsumer<Request> consumer) {
while (pendingRequestCount.get() >= maxPendingRequests) {
try {
Thread.sleep(pollInterval);
} catch (InterruptedException e) {
log.trace("Failed to wait until the server has capacity to handle new requests", e);
}
}
requests.forEach(request -> {
long currentTime = System.currentTimeMillis();
long expireTs = bytesToLong(request.getHeaders().get(EXPIRE_TS_HEADER));
if (expireTs >= currentTime) {
byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER);
if (requestIdHeader == null) {
log.error("[{}] Missing requestId in header", request);
return;
}
byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER);
if (responseTopicHeader == null) {
log.error("[{}] Missing response topic in header", request);
return;
}
UUID requestId = bytesToUuid(requestIdHeader);
String responseTopic = bytesToString(responseTopicHeader);
try {
pendingRequestCount.getAndIncrement();
stats.incrementTotal();
AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request),
response -> {
pendingRequestCount.decrementAndGet();
response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId));
responseProducer.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null);
stats.incrementSuccessful();
},
e -> {
pendingRequestCount.decrementAndGet();
if (e.getCause() != null && e.getCause() instanceof TimeoutException) {
log.warn("[{}] Timeout to process the request: {}", requestId, request, e);
} else {
log.trace("[{}] Failed to process the request: {}", requestId, request, e);
}
stats.incrementFailed();
},
requestTimeout,
scheduler,
callbackExecutor);
} catch (Throwable e) {
pendingRequestCount.decrementAndGet();
log.warn("[{}] Failed to process the request: {}", requestId, request, e);
stats.incrementFailed();
}
}
});
consumer.commit();
}
public void subscribe(Set<TopicPartitionInfo> partitions) {
requestConsumer.update(partitions);
}
public void stop() {
if (requestConsumer != null) {
requestConsumer.stop();
requestConsumer.awaitStop();
}
if (responseProducer != null) {
responseProducer.stop();
}
if (scheduler != null) {
scheduler.shutdownNow();
}
}
}

5
common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java

@ -25,7 +25,6 @@ import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdateConfigTask;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdatePartitionsTask;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate;
import java.util.Collection;
@ -50,7 +49,7 @@ import java.util.function.Function;
public class MainQueueConsumerManager<M extends TbQueueMsg, C extends QueueConfig> {
@Getter
protected final QueueKey queueKey;
protected final Object queueKey;
@Getter
protected C config;
protected final MsgPackProcessor<M, C> msgPackProcessor;
@ -72,7 +71,7 @@ public class MainQueueConsumerManager<M extends TbQueueMsg, C extends QueueConfi
protected volatile boolean stopped;
@Builder
public MainQueueConsumerManager(QueueKey queueKey, C config,
public MainQueueConsumerManager(Object queueKey, C config,
MsgPackProcessor<M, C> msgPackProcessor,
BiFunction<C, TopicPartitionInfo, TbQueueConsumer<M>> consumerCreator,
ExecutorService consumerExecutor,

3
common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java

@ -26,7 +26,6 @@ import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.AddPartitionsTask;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.DeletePartitionsTask;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.RemovePartitionsTask;
import org.thingsboard.server.queue.discovery.QueueKey;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@ -44,7 +43,7 @@ public class PartitionedQueueConsumerManager<M extends TbQueueMsg> extends MainQ
private final String topic;
@Builder(builderMethodName = "create") // not to conflict with super.builder()
public PartitionedQueueConsumerManager(QueueKey queueKey, String topic, long pollInterval, MsgPackProcessor<M, QueueConfig> msgPackProcessor,
public PartitionedQueueConsumerManager(Object queueKey, String topic, long pollInterval, MsgPackProcessor<M, QueueConfig> msgPackProcessor,
BiFunction<QueueConfig, TopicPartitionInfo, TbQueueConsumer<M>> consumerCreator, TbQueueAdmin queueAdmin,
ExecutorService consumerExecutor, ScheduledExecutorService scheduler,
ExecutorService taskExecutor, Consumer<Throwable> uncaughtErrorHandler) {

4
common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java

@ -18,10 +18,12 @@ package org.thingsboard.server.queue.common.state;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import java.util.Collections;
public class DefaultQueueStateService<E extends TbQueueMsg, S extends TbQueueMsg> extends QueueStateService<E, S> {
public DefaultQueueStateService(PartitionedQueueConsumerManager<E> eventConsumer) {
super(eventConsumer);
super(eventConsumer, Collections.emptyList());
}
}

16
common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java

@ -22,8 +22,11 @@ import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager;
import org.thingsboard.server.queue.discovery.QueueKey;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic;
@ -34,17 +37,20 @@ public class KafkaQueueStateService<E extends TbQueueMsg, S extends TbQueueMsg>
private final PartitionedQueueConsumerManager<S> stateConsumer;
private final Supplier<Map<String, Long>> eventsStartOffsetsProvider;
private final Set<TopicPartitionInfo> partitionsInProgress = ConcurrentHashMap.newKeySet();
@Builder
public KafkaQueueStateService(PartitionedQueueConsumerManager<E> eventConsumer,
PartitionedQueueConsumerManager<S> stateConsumer,
List<PartitionedQueueConsumerManager<?>> otherConsumers,
Supplier<Map<String, Long>> eventsStartOffsetsProvider) {
super(eventConsumer);
super(eventConsumer, otherConsumers != null ? otherConsumers : Collections.emptyList());
this.stateConsumer = stateConsumer;
this.eventsStartOffsetsProvider = eventsStartOffsetsProvider;
}
@Override
protected void addPartitions(QueueKey queueKey, Set<TopicPartitionInfo> partitions) {
protected void addPartitions(QueueKey queueKey, Set<TopicPartitionInfo> partitions, Runnable whenAllProcessed) {
Map<String, Long> eventsStartOffsets = eventsStartOffsetsProvider != null ? eventsStartOffsetsProvider.get() : null; // remembering the offsets before subscribing to states
Set<TopicPartitionInfo> statePartitions = withTopic(partitions, stateConsumer.getTopic());
@ -57,11 +63,17 @@ public class KafkaQueueStateService<E extends TbQueueMsg, S extends TbQueueMsg>
log.info("Finished partition {} (still in progress: {})", statePartition, partitionsInProgress);
if (partitionsInProgress.isEmpty()) {
log.info("All partitions processed");
if (whenAllProcessed != null) {
whenAllProcessed.run();
}
}
TopicPartitionInfo eventPartition = statePartition.withTopic(eventConsumer.getTopic());
if (this.partitions.get(queueKey).contains(eventPartition)) {
eventConsumer.addPartitions(Set.of(eventPartition), null, eventsStartOffsets != null ? eventsStartOffsets::get : null);
for (PartitionedQueueConsumerManager<?> consumer : otherConsumers) {
consumer.addPartitions(Set.of(statePartition.withTopic(consumer.getTopic())));
}
}
} finally {
readLock.unlock();

35
common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java

@ -25,9 +25,9 @@ import org.thingsboard.server.queue.discovery.QueueKey;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@ -37,19 +37,19 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop
public abstract class QueueStateService<E extends TbQueueMsg, S extends TbQueueMsg> {
protected final PartitionedQueueConsumerManager<E> eventConsumer;
protected final List<PartitionedQueueConsumerManager<?>> otherConsumers;
@Getter
protected final Map<QueueKey, Set<TopicPartitionInfo>> partitions = new HashMap<>();
protected final Set<TopicPartitionInfo> partitionsInProgress = ConcurrentHashMap.newKeySet();
protected boolean initialized;
protected final ReadWriteLock partitionsLock = new ReentrantReadWriteLock();
protected QueueStateService(PartitionedQueueConsumerManager<E> eventConsumer) {
protected QueueStateService(PartitionedQueueConsumerManager<E> eventConsumer, List<PartitionedQueueConsumerManager<?>> otherConsumers) {
this.eventConsumer = eventConsumer;
this.otherConsumers = otherConsumers;
}
public void update(QueueKey queueKey, Set<TopicPartitionInfo> newPartitions) {
public void update(QueueKey queueKey, Set<TopicPartitionInfo> newPartitions, Runnable whenAllProcessed) {
newPartitions = withTopic(newPartitions, eventConsumer.getTopic());
var writeLock = partitionsLock.writeLock();
writeLock.lock();
@ -71,17 +71,29 @@ public abstract class QueueStateService<E extends TbQueueMsg, S extends TbQueueM
}
if (!addedPartitions.isEmpty()) {
addPartitions(queueKey, addedPartitions);
addPartitions(queueKey, addedPartitions, whenAllProcessed);
} else {
if (whenAllProcessed != null) {
whenAllProcessed.run();
}
}
initialized = true;
}
protected void addPartitions(QueueKey queueKey, Set<TopicPartitionInfo> partitions) {
protected void addPartitions(QueueKey queueKey, Set<TopicPartitionInfo> partitions, Runnable whenAllProcessed) {
if (whenAllProcessed != null) {
whenAllProcessed.run();
}
eventConsumer.addPartitions(partitions);
for (PartitionedQueueConsumerManager<?> consumer : otherConsumers) {
consumer.addPartitions(withTopic(partitions, consumer.getTopic()));
}
}
protected void removePartitions(QueueKey queueKey, Set<TopicPartitionInfo> partitions) {
eventConsumer.removePartitions(partitions);
for (PartitionedQueueConsumerManager<?> consumer : otherConsumers) {
consumer.removePartitions(withTopic(partitions, consumer.getTopic()));
}
}
public void delete(Set<TopicPartitionInfo> partitions) {
@ -100,10 +112,9 @@ public abstract class QueueStateService<E extends TbQueueMsg, S extends TbQueueM
protected void deletePartitions(Set<TopicPartitionInfo> partitions) {
eventConsumer.delete(withTopic(partitions, eventConsumer.getTopic()));
}
public Set<TopicPartitionInfo> getPartitionsInProgress() {
return initialized ? partitionsInProgress : null;
for (PartitionedQueueConsumerManager<?> consumer : otherConsumers) {
consumer.removePartitions(withTopic(partitions, consumer.getTopic()));
}
}
public void stop() {

11
common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java

@ -68,6 +68,8 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
private List<ServiceType> serviceTypes;
private ServiceInfo serviceInfo;
private boolean ready = true;
@PostConstruct
public void init() {
if (StringUtils.isEmpty(serviceId)) {
@ -85,6 +87,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
assignedTenantProfiles = Collections.emptySet();
}
if (serviceTypes.contains(ServiceType.EDQS)) {
ready = false;
if (StringUtils.isBlank(edqsConfig.getLabel())) {
edqsConfig.setLabel(serviceId);
}
@ -131,9 +134,17 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList()));
}
builder.setLabel(edqsConfig.getLabel());
builder.setReady(ready);
return serviceInfo = builder.build();
}
@Override
public boolean setReady(boolean ready) {
boolean changed = this.ready != ready;
this.ready = ready;
return changed;
}
private TransportProtos.SystemInfoProto getCurrentSystemInfoProto() {
TransportProtos.SystemInfoProto.Builder builder = TransportProtos.SystemInfoProto.newBuilder();

2
common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java

@ -25,4 +25,6 @@ public interface DiscoveryService {
boolean isMonolith();
void setReady(boolean ready);
}

9
common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java

@ -55,4 +55,13 @@ public class DummyDiscoveryService implements DiscoveryService {
public boolean isMonolith() {
return true;
}
@Override
public void setReady(boolean ready) {
boolean changed = serviceInfoProvider.setReady(ready);
if (changed) {
serviceInfoProvider.generateNewServiceInfoWithCurrentSystemInfo();
}
}
}

2
common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java

@ -156,7 +156,7 @@ public class HashPartitionService implements PartitionService {
@Override
public String getTopic(QueueKey queueKey) {
return partitionTopicsMap.get(queueKey);
return topicService.buildTopicName(partitionTopicsMap.get(queueKey));
}
private void doInitRuleEnginePartitions() {

13
common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java

@ -177,6 +177,19 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi
}
}
@Override
public void setReady(boolean ready) {
log.debug("Marking current service as {}", ready ? "ready" : "NOT ready");
boolean changed = serviceInfoProvider.setReady(ready);
if (changed) {
try {
publishCurrentServer();
} catch (Exception e) {
log.error("Failed to update server readiness status", e);
}
}
}
private boolean currentServerExists() {
if (nodePath == null) {
return false;

4
common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java

@ -44,6 +44,10 @@ public class EdqsConfig {
private int maxPendingRequests;
@Value("${queue.edqs.max_request_timeout:20000}")
private int maxRequestTimeout;
@Value("${queue.edqs.request_executor_size:50}")
private int requestExecutorSize;
@Value("${queue.edqs.versions_cache_ttl:60}")
private int versionsCacheTtl;
public String getLabel() {
if (partitioningStrategy == EdqsPartitioningStrategy.NONE) {

70
common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsExecutors.java

@ -0,0 +1,70 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.edqs;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@Lazy
@Component
@Getter
@RequiredArgsConstructor
public class EdqsExecutors {
private final EdqsConfig edqsConfig;
private ExecutorService consumersExecutor;
private ExecutorService consumerTaskExecutor;
private ScheduledExecutorService scheduler;
private ListeningExecutorService requestExecutor;
@PostConstruct
private void init() {
consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer"));
consumerTaskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor");
scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler");
requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(edqsConfig.getRequestExecutorSize(), "edqs-requests"));
}
@PreDestroy
private void destroy() {
if (consumersExecutor != null) {
consumersExecutor.shutdownNow();
}
if (consumerTaskExecutor != null) {
consumerTaskExecutor.shutdownNow();
}
if (scheduler != null) {
scheduler.shutdownNow();
}
if (requestExecutor != null) {
requestExecutor.shutdownNow();
}
}
}

5
common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java

@ -19,8 +19,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueHandler;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueResponseTemplate;
import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
public interface EdqsQueueFactory {
@ -33,7 +34,7 @@ public interface EdqsQueueFactory {
TbQueueProducer<TbProtoQueueMsg<ToEdqsMsg>> createEdqsStateProducer();
TbQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate();
PartitionedQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate(TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> handler);
TbQueueAdmin getEdqsQueueAdmin();

26
common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java

@ -17,16 +17,15 @@ package org.thingsboard.server.queue.edqs;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueHandler;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueResponseTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate;
import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer;
@ -39,6 +38,7 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory {
private final InMemoryStorage storage;
private final EdqsConfig edqsConfig;
private final EdqsExecutors edqsExecutors;
private final StatsFactory statsFactory;
private final TbQueueAdmin queueAdmin;
@ -63,17 +63,21 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory {
}
@Override
public TbQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate() {
TbQueueConsumer<TbProtoQueueMsg<ToEdqsMsg>> requestConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic());
public PartitionedQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate(TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> handler) {
TbQueueProducer<TbProtoQueueMsg<FromEdqsMsg>> responseProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getResponsesTopic());
return DefaultTbQueueResponseTemplate.<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>>builder()
.requestTemplate(requestConsumer)
.responseTemplate(responseProducer)
.maxPendingRequests(edqsConfig.getMaxPendingRequests())
.requestTimeout(edqsConfig.getMaxRequestTimeout())
return PartitionedQueueResponseTemplate.<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>>builder()
.key("edqs")
.handler(handler)
.requestsTopic(edqsConfig.getRequestsTopic())
.consumerCreator(tpi -> new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic()))
.responseProducer(responseProducer)
.pollInterval(edqsConfig.getPollInterval())
.requestTimeout(edqsConfig.getMaxRequestTimeout())
.maxPendingRequests(edqsConfig.getMaxPendingRequests())
.consumerExecutor(edqsExecutors.getConsumersExecutor())
.callbackExecutor(edqsExecutors.getRequestExecutor())
.consumerTaskExecutor(edqsExecutors.getConsumerTaskExecutor())
.stats(statsFactory.createMessagesStats(StatsType.EDQS.getName()))
.executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs"))
.build();
}

35
common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java

@ -16,14 +16,13 @@
package org.thingsboard.server.queue.edqs;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.stats.StatsType;
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg;
import org.thingsboard.server.queue.TbQueueHandler;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueResponseTemplate;
import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate;
import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
@ -45,6 +44,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory {
private final TbKafkaAdmin edqsRequestsAdmin;
private final TbKafkaAdmin edqsStateAdmin;
private final EdqsConfig edqsConfig;
private final EdqsExecutors edqsExecutors;
private final TbServiceInfoProvider serviceInfoProvider;
private final TbKafkaConsumerStatsService consumerStatsService;
private final TopicService topicService;
@ -53,7 +53,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory {
private final AtomicInteger consumerCounter = new AtomicInteger();
public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs,
EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider,
EdqsConfig edqsConfig, EdqsExecutors edqsExecutors, TbServiceInfoProvider serviceInfoProvider,
TbKafkaConsumerStatsService consumerStatsService, TopicService topicService,
StatsFactory statsFactory) {
this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs());
@ -61,6 +61,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory {
this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs());
this.kafkaSettings = kafkaSettings;
this.edqsConfig = edqsConfig;
this.edqsExecutors = edqsExecutors;
this.serviceInfoProvider = serviceInfoProvider;
this.consumerStatsService = consumerStatsService;
this.topicService = topicService;
@ -116,25 +117,29 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory {
}
@Override
public TbQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate() {
var requestConsumer = createEdqsMsgConsumer(edqsConfig.getRequestsTopic(),
"edqs-requests-consumer-" + serviceInfoProvider.getServiceId(),
"edqs-requests-consumer-group",
false, edqsRequestsAdmin);
public PartitionedQueueResponseTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> createEdqsResponseTemplate(TbQueueHandler<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> handler) {
var responseProducer = TbKafkaProducerTemplate.<TbProtoQueueMsg<FromEdqsMsg>>builder()
.settings(kafkaSettings)
.clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(edqsConfig.getResponsesTopic()))
.admin(edqsRequestsAdmin)
.build();
return DefaultTbQueueResponseTemplate.<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>>builder()
.requestTemplate(requestConsumer)
.responseTemplate(responseProducer)
.maxPendingRequests(edqsConfig.getMaxPendingRequests())
.requestTimeout(edqsConfig.getMaxRequestTimeout())
return PartitionedQueueResponseTemplate.<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>>builder()
.key("edqs")
.handler(handler)
.requestsTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic()))
.consumerCreator(tpi -> createEdqsMsgConsumer(edqsConfig.getRequestsTopic(),
"edqs-requests-consumer-" + serviceInfoProvider.getServiceId() + "-" + tpi.getPartition().orElse(999),
"edqs-requests-consumer-group",
false, edqsRequestsAdmin))
.responseProducer(responseProducer)
.pollInterval(edqsConfig.getPollInterval())
.requestTimeout(edqsConfig.getMaxRequestTimeout())
.maxPendingRequests(edqsConfig.getMaxPendingRequests())
.consumerExecutor(edqsExecutors.getConsumersExecutor())
.callbackExecutor(edqsExecutors.getRequestExecutor())
.consumerTaskExecutor(edqsExecutors.getConsumerTaskExecutor())
.stats(statsFactory.createMessagesStats(StatsType.EDQS.getName()))
.executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs"))
.build();
}

3
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java

@ -93,8 +93,7 @@ public class TbKafkaConsumerStatsService {
log.info("[{}] Topic partitions with lag: [{}].", groupId, builder.toString());
}
} catch (Exception e) {
log.warn("[{}] Failed to get consumer group stats. Reason - {}.", groupId, e.getMessage());
log.trace("Detailed error: ", e);
log.warn("[{}] Failed to get consumer group stats", groupId, e);
}
}

4
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java

@ -75,10 +75,6 @@ public class TbKafkaProducerTemplate<T extends TbQueueMsg> implements TbQueuePro
topics = ConcurrentHashMap.newKeySet();
}
@Override
public void init() {
}
void addAnalyticHeaders(List<Header> headers) {
headers.add(new RecordHeader("_producerId", getClientId().getBytes(StandardCharsets.UTF_8)));
headers.add(new RecordHeader("_threadName", Thread.currentThread().getName().getBytes(StandardCharsets.UTF_8)));

5
common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java

@ -33,11 +33,6 @@ public class InMemoryTbQueueProducer<T extends TbQueueMsg> implements TbQueuePro
this.defaultTopic = defaultTopic;
}
@Override
public void init() {
}
@Override
public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) {
boolean result = storage.put(tpi.getFullTopicName(), msg);

2
common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java

@ -131,7 +131,7 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient {
report.forEach((parent, statsMsg) -> {
try {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, parent.getTenantId(), parent.getId())
.newByTopic(msgProducer.getDefaultTopic());
.withTopic(msgProducer.getDefaultTopic());
reportStatsPerTpi.computeIfAbsent(tpi, k -> new ArrayList<>()).add(statsMsg.build());
} catch (TenantNotFoundException e) {
log.debug("Couldn't report usage stats for non-existing tenant: {}", e.getTenantId());

19
common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java

@ -37,11 +37,23 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThan;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.BDDMockito.*;
import static org.mockito.Mockito.*;
import static org.mockito.BDDMockito.RETURNS_DEEP_STUBS;
import static org.mockito.BDDMockito.atLeastOnce;
import static org.mockito.BDDMockito.lenient;
import static org.mockito.BDDMockito.mock;
import static org.mockito.BDDMockito.never;
import static org.mockito.BDDMockito.spy;
import static org.mockito.BDDMockito.times;
import static org.mockito.BDDMockito.verify;
import static org.mockito.BDDMockito.willAnswer;
import static org.mockito.BDDMockito.willDoNothing;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.hamcrest.MockitoHamcrest.longThat;
@Slf4j
@ -96,7 +108,6 @@ public class DefaultTbQueueRequestTemplateTest {
inst.init();
assertThat(inst.nextCleanupNs, equalTo(0L));
verify(queueAdmin, times(1)).createTopicIfNotExists(topic);
verify(requestTemplate, times(1)).init();
verify(responseTemplate, times(1)).subscribe();
verify(executorMock, times(1)).submit(any(Runnable.class));

5
common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java

@ -24,9 +24,12 @@ public class TbelCfCtx implements TbelCfObject {
@Getter
private final Map<String, TbelCfArg> args;
@Getter
private final long msgTs;
public TbelCfCtx(Map<String, TbelCfArg> args) {
public TbelCfCtx(Map<String, TbelCfArg> args, long lastUpdateTs) {
this.args = Collections.unmodifiableMap(args);
this.msgTs = lastUpdateTs != -1 ? lastUpdateTs : System.currentTimeMillis();
}
@Override

10
dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java

@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.query.EntityTypeFilter;
import org.thingsboard.server.common.data.query.KeyFilter;
import org.thingsboard.server.common.data.query.RelationsQueryFilter;
import org.thingsboard.server.common.msg.edqs.EdqsApiService;
import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.common.stats.EdqsStatsService;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
@ -88,6 +89,9 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
@Lazy
EntityServiceRegistry entityServiceRegistry;
@Autowired
private EdqsService edqsService;
@Autowired
@Lazy
private EdqsApiService edqsApiService;
@ -104,7 +108,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
long startNs = System.nanoTime();
Long result;
if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) {
if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) {
EdqsRequest request = EdqsRequest.builder()
.entityCountQuery(query)
.build();
@ -126,7 +130,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
long startNs = System.nanoTime();
PageData<EntityData> result;
if (edqsApiService.isEnabled() && validForEdqs(query)) {
if (edqsService.isApiEnabled() && validForEdqs(query)) {
EdqsRequest request = EdqsRequest.builder()
.entityDataQuery(query)
.build();
@ -295,7 +299,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
}
if ((query.getEntityFields() == null || query.getEntityFields().isEmpty()) &&
(query.getLatestValues() == null || query.getLatestValues().isEmpty())) {
(query.getLatestValues() == null || query.getLatestValues().isEmpty())) {
return false;
}

28
dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java

@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.Abstrac
import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBootstrapServerCredential;
import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.RPKLwM2MBootstrapServerCredential;
import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.X509LwM2MBootstrapServerCredential;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.rule.RuleChain;
@ -188,13 +189,11 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<D
}
if (deviceProfile.getDefaultRuleChainId() != null) {
RuleChain ruleChain = ruleChainService.findRuleChainById(tenantId, deviceProfile.getDefaultRuleChainId());
if (ruleChain == null) {
throw new DataValidationException("Can't assign non-existent rule chain!");
}
if (!ruleChain.getTenantId().equals(deviceProfile.getTenantId())) {
throw new DataValidationException("Can't assign rule chain from different tenant!");
}
validateRuleChain(tenantId, deviceProfile.getTenantId(), deviceProfile.getDefaultRuleChainId());
}
if (deviceProfile.getDefaultEdgeRuleChainId() != null) {
validateRuleChain(tenantId, deviceProfile.getTenantId(), deviceProfile.getDefaultEdgeRuleChainId());
}
if (deviceProfile.getDefaultDashboardId() != null) {
@ -210,6 +209,16 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<D
validateOtaPackage(tenantId, deviceProfile, deviceProfile.getId());
}
private void validateRuleChain(TenantId tenantId, TenantId deviceProfileTenantId, RuleChainId ruleChainId) {
RuleChain ruleChain = ruleChainService.findRuleChainById(tenantId, ruleChainId);
if (ruleChain == null) {
throw new DataValidationException("Can't assign non-existent rule chain!");
}
if (!ruleChain.getTenantId().equals(deviceProfileTenantId)) {
throw new DataValidationException("Can't assign rule chain from different tenant!");
}
}
@Override
protected DeviceProfile validateUpdate(TenantId tenantId, DeviceProfile deviceProfile) {
DeviceProfile old = deviceProfileDao.findById(deviceProfile.getTenantId(), deviceProfile.getId().getId());
@ -329,8 +338,8 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<D
}
if (serverConfig.getShortServerId() != null) {
if (serverConfig.isBootstrapServerIs()){
if(serverConfig.getShortServerId() < 0 || serverConfig.getShortServerId() > 65535){
if (serverConfig.isBootstrapServerIs()) {
if (serverConfig.getShortServerId() < 0 || serverConfig.getShortServerId() > 65535) {
throw new DeviceCredentialsValidationException("Bootstrap Server ShortServerId must be in range [0 - 65535]!");
}
} else {
@ -423,4 +432,5 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator<D
private String getCertificateString(X509Certificate cert) throws CertificateEncodingException {
return EncryptionUtil.certTrimNewLines(Base64.getEncoder().encodeToString(cert.getEncoded()));
}
}

16
dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java

@ -44,17 +44,10 @@ public interface EdgeRepository extends JpaRepository<EdgeEntity, UUID> {
"WHERE d.id = :edgeId")
EdgeInfoEntity findEdgeInfoById(@Param("edgeId") UUID edgeId);
@Query(value = "SELECT ee.id, ee.created_time, ee.additional_info, ee.customer_id, " +
"ee.root_rule_chain_id, ee.type, ee.name, ee.label, ee.routing_key, " +
"ee.secret, ee.tenant_id, ee.version " +
"FROM edge ee " +
"JOIN attribute_kv ON ee.id = attribute_kv.entity_id " +
"JOIN key_dictionary ON attribute_kv.attribute_key = key_dictionary.key_id " +
"WHERE attribute_kv.bool_v = true AND key_dictionary.key = 'active' " +
"AND (:textSearch IS NULL OR ee.name ILIKE CONCAT('%', :textSearch, '%')) " +
"ORDER BY ee.id", nativeQuery = true)
Page<EdgeEntity> findActiveEdges(@Param("textSearch") String textSearch,
Pageable pageable);
@Query(value = "SELECT * FROM edge_active_attribute_view edge_active",
countQuery = "SELECT count(*) FROM edge_active_attribute_view",
nativeQuery = true)
Page<EdgeEntity> findActiveEdges(Pageable pageable);
@Query("SELECT d.id FROM EdgeEntity d WHERE d.tenantId = :tenantId " +
"AND (:textSearch IS NULL OR ilike(d.name, CONCAT('%', :textSearch, '%')) = true)")
@ -171,4 +164,5 @@ public interface EdgeRepository extends JpaRepository<EdgeEntity, UUID> {
@Query("SELECT new org.thingsboard.server.common.data.edqs.fields.EdgeFields(e.id, e.createdTime, e.tenantId, e.customerId," +
"e.name, e.version, e.type, e.label, e.additionalInfo) FROM EdgeEntity e WHERE e.id > :id ORDER BY e.id")
List<EdgeFields> findNextBatch(@Param("id") UUID id, Limit limit);
}

10
dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java

@ -44,7 +44,7 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.dao.util.SqlDao;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.UUID;
@ -58,6 +58,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@RequiredArgsConstructor
@Slf4j
public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao<EdgeEventEntity, EdgeEvent> implements EdgeEventDao {
private static final List<SortOrder> SORT_ORDERS = Collections.singletonList(new SortOrder("seqId"));
private final UUID systemTenantId = NULL_UUID;
@ -175,11 +176,6 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao<EdgeEventEnti
@Override
public PageData<EdgeEvent> findEdgeEvents(UUID tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) {
List<SortOrder> sortOrders = new ArrayList<>();
if (pageLink.getSortOrder() != null) {
sortOrders.add(pageLink.getSortOrder());
}
sortOrders.add(new SortOrder("seqId"));
return DaoUtil.toPageData(
edgeEventRepository
.findEdgeEventsByTenantIdAndEdgeId(
@ -190,7 +186,7 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao<EdgeEventEnti
pageLink.getEndTime(),
seqIdStart,
seqIdEnd,
DaoUtil.toPageable(pageLink, sortOrders)));
DaoUtil.toPageable(pageLink, SORT_ORDERS)));
}
@Override

5
dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java

@ -68,10 +68,7 @@ public class JpaEdgeDao extends JpaAbstractDao<EdgeEntity, Edge> implements Edge
@Override
public PageData<Edge> findActiveEdges(PageLink pageLink) {
return DaoUtil.toPageData(
edgeRepository.findActiveEdges(
pageLink.getTextSearch(),
DaoUtil.toPageable(pageLink)));
return DaoUtil.toPageData(edgeRepository.findActiveEdges(DaoUtil.toPageable(pageLink)));
}
@Override

15
dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java

@ -35,24 +35,9 @@ public class DummyEdqsApiService implements EdqsApiService {
throw new UnsupportedOperationException();
}
@Override
public boolean isEnabled() {
return false;
}
@Override
public void setEnabled(boolean enabled) {
log.warn("Got request to enable EDQS API, but it isn't supported", new RuntimeException("stacktrace"));
}
@Override
public boolean isSupported() {
return false;
}
@Override
public boolean isAutoEnable() {
return false;
}
}

11
dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java

@ -19,6 +19,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.ObjectType;
import org.thingsboard.server.common.data.edqs.EdqsObject;
import org.thingsboard.server.common.data.edqs.EdqsState;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.thingsboard.server.common.data.id.EntityId;
@ -47,4 +48,14 @@ public class DummyEdqsService implements EdqsService {
@Override
public void processSystemMsg(ToCoreEdqsMsg request) {}
@Override
public boolean isApiEnabled() {
return getState().isApiEnabled();
}
@Override
public EdqsState getState() {
return new EdqsState();
}
}

20
dao/src/main/resources/sql/schema-views-and-functions.sql

@ -72,6 +72,26 @@ u.first_name as assignee_first_name, u.last_name as assignee_last_name, u.email
FROM alarm a
LEFT JOIN tb_user u ON u.id = a.assignee_id;
DROP VIEW IF EXISTS edge_active_attribute_view CASCADE;
CREATE OR REPLACE VIEW edge_active_attribute_view AS
SELECT ee.id
, ee.created_time
, ee.additional_info
, ee.customer_id
, ee.root_rule_chain_id
, ee.type
, ee.name
, ee.label
, ee.routing_key
, ee.secret
, ee.tenant_id
, ee.version
FROM edge ee
JOIN attribute_kv ON ee.id = attribute_kv.entity_id
JOIN key_dictionary ON attribute_kv.attribute_key = key_dictionary.key_id
WHERE attribute_kv.bool_v = true AND key_dictionary.key = 'active'
ORDER BY ee.id;
CREATE OR REPLACE FUNCTION create_or_update_active_alarm(
t_id uuid, c_id uuid, a_id uuid, a_created_ts bigint,
a_o_id uuid, a_o_type integer, a_type varchar,

25
dao/src/test/java/org/thingsboard/server/dao/service/EdgeEventServiceTest.java

@ -16,7 +16,6 @@
package org.thingsboard.server.dao.service;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Assert;
import org.junit.Before;
@ -38,8 +37,6 @@ import org.thingsboard.server.dao.edge.EdgeEventService;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import static org.apache.commons.lang3.time.DateFormatUtils.ISO_8601_EXTENDED_DATETIME_FORMAT;
@ -103,26 +100,23 @@ public class EdgeEventServiceTest extends AbstractServiceTest {
}
@Test
public void findEdgeEventsByTimeDescOrder() throws Exception {
public void findEdgeEventsBySeqIdOrder_createdTimeOrderIgnored() throws Exception {
EdgeId edgeId = new EdgeId(Uuids.timeBased());
DeviceId deviceId = new DeviceId(Uuids.timeBased());
List<ListenableFuture<Void>> futures = new ArrayList<>();
futures.add(saveEdgeEventWithProvidedTime(timeBeforeStartTime, edgeId, deviceId, tenantId));
futures.add(saveEdgeEventWithProvidedTime(eventTime, edgeId, deviceId, tenantId));
futures.add(saveEdgeEventWithProvidedTime(eventTime + 1, edgeId, deviceId, tenantId));
futures.add(saveEdgeEventWithProvidedTime(eventTime + 2, edgeId, deviceId, tenantId));
futures.add(saveEdgeEventWithProvidedTime(timeAfterEndTime, edgeId, deviceId, tenantId));
Futures.allAsList(futures).get();
saveEdgeEventWithProvidedTime(timeBeforeStartTime, edgeId, deviceId, tenantId).get();
saveEdgeEventWithProvidedTime(eventTime, edgeId, deviceId, tenantId).get();
saveEdgeEventWithProvidedTime(eventTime + 2, edgeId, deviceId, tenantId).get();
saveEdgeEventWithProvidedTime(eventTime + 1, edgeId, deviceId, tenantId).get();
saveEdgeEventWithProvidedTime(timeAfterEndTime, edgeId, deviceId, tenantId).get();
TimePageLink pageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime);
PageData<EdgeEvent> edgeEvents = edgeEventService.findEdgeEvents(tenantId, edgeId, 0L, null, pageLink);
Assert.assertNotNull(edgeEvents.getData());
Assert.assertEquals(2, edgeEvents.getData().size());
Assert.assertEquals(Uuids.startOf(eventTime + 2), edgeEvents.getData().get(0).getUuidId());
Assert.assertEquals(Uuids.startOf(eventTime + 1), edgeEvents.getData().get(1).getUuidId());
Assert.assertEquals(Uuids.startOf(eventTime), edgeEvents.getData().get(0).getUuidId());
Assert.assertEquals(Uuids.startOf(eventTime + 2), edgeEvents.getData().get(1).getUuidId());
Assert.assertTrue(edgeEvents.hasNext());
Assert.assertNotNull(pageLink.nextPageLink());
@ -130,7 +124,7 @@ public class EdgeEventServiceTest extends AbstractServiceTest {
Assert.assertNotNull(edgeEvents.getData());
Assert.assertEquals(1, edgeEvents.getData().size());
Assert.assertEquals(Uuids.startOf(eventTime), edgeEvents.getData().get(0).getUuidId());
Assert.assertEquals(Uuids.startOf(eventTime + 1), edgeEvents.getData().get(0).getUuidId());
Assert.assertFalse(edgeEvents.hasNext());
edgeEventDao.cleanupEvents(1);
@ -141,4 +135,5 @@ public class EdgeEventServiceTest extends AbstractServiceTest {
edgeEvent.setId(new EdgeEventId(Uuids.startOf(time)));
return edgeEventService.saveAsync(edgeEvent);
}
}

1
dao/src/test/resources/sql/psql/drop-all-tables.sql

@ -14,6 +14,7 @@ DROP VIEW IF EXISTS device_info_active_attribute_view CASCADE;
DROP VIEW IF EXISTS device_info_active_ts_view CASCADE;
DROP VIEW IF EXISTS device_info_view CASCADE;
DROP VIEW IF EXISTS alarm_info CASCADE;
DROP VIEW IF EXISTS edge_acitve_attribute_view CASCADE;
DROP TABLE IF EXISTS admin_settings;
DROP TABLE IF EXISTS entity_alarm;

14
docker/compose-utils.sh

@ -42,20 +42,8 @@ function additionalComposeQueueArgs() {
confluent)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.confluent.yml"
;;
aws-sqs)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.aws-sqs.yml"
;;
pubsub)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.pubsub.yml"
;;
rabbitmq)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.rabbitmq.yml"
;;
service-bus)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml"
;;
*)
echo "Unknown Queue service TB_QUEUE_TYPE value specified in the .env file: '${TB_QUEUE_TYPE}'. Should be either 'kafka' or 'confluent' or 'aws-sqs' or 'pubsub' or 'rabbitmq' or 'service-bus'." >&2
echo "Unknown Queue service TB_QUEUE_TYPE value specified in the .env file: '${TB_QUEUE_TYPE}'. Should be either 'kafka' or 'confluent'." >&2
exit 1
esac
echo $ADDITIONAL_COMPOSE_QUEUE_ARGS

61
docker/docker-compose.aws-sqs.yml

@ -1,61 +0,0 @@
#
# Copyright © 2016-2025 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '3.0'
services:
tb-js-executor:
env_file:
- queue-aws-sqs.env
tb-core1:
env_file:
- queue-aws-sqs.env
tb-core2:
env_file:
- queue-aws-sqs.env
tb-rule-engine1:
env_file:
- queue-aws-sqs.env
tb-rule-engine2:
env_file:
- queue-aws-sqs.env
tb-mqtt-transport1:
env_file:
- queue-aws-sqs.env
tb-mqtt-transport2:
env_file:
- queue-aws-sqs.env
tb-http-transport1:
env_file:
- queue-aws-sqs.env
tb-http-transport2:
env_file:
- queue-aws-sqs.env
tb-coap-transport:
env_file:
- queue-aws-sqs.env
tb-lwm2m-transport:
env_file:
- queue-aws-sqs.env
tb-snmp-transport:
env_file:
- queue-aws-sqs.env
tb-vc-executor1:
env_file:
- queue-aws-sqs.env
tb-vc-executor2:
env_file:
- queue-aws-sqs.env

61
docker/docker-compose.pubsub.yml

@ -1,61 +0,0 @@
#
# Copyright © 2016-2025 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '3.0'
services:
tb-js-executor:
env_file:
- queue-pubsub.env
tb-core1:
env_file:
- queue-pubsub.env
tb-core2:
env_file:
- queue-pubsub.env
tb-rule-engine1:
env_file:
- queue-pubsub.env
tb-rule-engine2:
env_file:
- queue-pubsub.env
tb-mqtt-transport1:
env_file:
- queue-pubsub.env
tb-mqtt-transport2:
env_file:
- queue-pubsub.env
tb-http-transport1:
env_file:
- queue-pubsub.env
tb-http-transport2:
env_file:
- queue-pubsub.env
tb-coap-transport:
env_file:
- queue-pubsub.env
tb-lwm2m-transport:
env_file:
- queue-pubsub.env
tb-snmp-transport:
env_file:
- queue-pubsub.env
tb-vc-executor1:
env_file:
- queue-pubsub.env
tb-vc-executor2:
env_file:
- queue-pubsub.env

61
docker/docker-compose.rabbitmq.yml

@ -1,61 +0,0 @@
#
# Copyright © 2016-2025 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '3.0'
services:
tb-js-executor:
env_file:
- queue-rabbitmq.env
tb-core1:
env_file:
- queue-rabbitmq.env
tb-core2:
env_file:
- queue-rabbitmq.env
tb-rule-engine1:
env_file:
- queue-rabbitmq.env
tb-rule-engine2:
env_file:
- queue-rabbitmq.env
tb-mqtt-transport1:
env_file:
- queue-rabbitmq.env
tb-mqtt-transport2:
env_file:
- queue-rabbitmq.env
tb-http-transport1:
env_file:
- queue-rabbitmq.env
tb-http-transport2:
env_file:
- queue-rabbitmq.env
tb-coap-transport:
env_file:
- queue-rabbitmq.env
tb-lwm2m-transport:
env_file:
- queue-rabbitmq.env
tb-snmp-transport:
env_file:
- queue-rabbitmq.env
tb-vc-executor1:
env_file:
- queue-rabbitmq.env
tb-vc-executor2:
env_file:
- queue-rabbitmq.env

61
docker/docker-compose.service-bus.yml

@ -1,61 +0,0 @@
#
# Copyright © 2016-2025 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '3.0'
services:
tb-js-executor:
env_file:
- queue-service-bus.env
tb-core1:
env_file:
- queue-service-bus.env
tb-core2:
env_file:
- queue-service-bus.env
tb-rule-engine1:
env_file:
- queue-service-bus.env
tb-rule-engine2:
env_file:
- queue-service-bus.env
tb-mqtt-transport1:
env_file:
- queue-service-bus.env
tb-mqtt-transport2:
env_file:
- queue-service-bus.env
tb-http-transport1:
env_file:
- queue-service-bus.env
tb-http-transport2:
env_file:
- queue-service-bus.env
tb-coap-transport:
env_file:
- queue-service-bus.env
tb-lwm2m-transport:
env_file:
- queue-service-bus.env
tb-snmp-transport:
env_file:
- queue-service-bus.env
tb-vc-executor1:
env_file:
- queue-service-bus.env
tb-vc-executor2:
env_file:
- queue-service-bus.env

4
docker/queue-aws-sqs.env

@ -1,4 +0,0 @@
TB_QUEUE_TYPE=aws-sqs
TB_QUEUE_AWS_SQS_ACCESS_KEY_ID=YOUR_KEY
TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY=YOUR_SECRET
TB_QUEUE_AWS_SQS_REGION=YOUR_REGION

3
docker/queue-pubsub.env

@ -1,3 +0,0 @@
TB_QUEUE_TYPE=pubsub
TB_QUEUE_PUBSUB_PROJECT_ID=YOUR_PROJECT_ID
TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save