From 3c496f474979f586686a8bb6900d2e57fe9cba7a Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 18 Aug 2022 00:43:54 +0300 Subject: [PATCH 01/50] Cassandra cloud connect to Datastax Astra DB using SECURE_BUNDLE, CLIENT_ID and CLIENT_SECRET --- .../install/ThingsboardInstallService.java | 8 ++++ .../install/CassandraKeyspaceService.java | 37 +++++++++++++++++++ .../service/install/NoSqlKeyspaceService.java | 19 ++++++++++ .../src/main/resources/thingsboard.yml | 8 ++++ .../transport/TransportNoSqlTestSuite.java | 1 + .../cassandra/AbstractCassandraCluster.java | 16 ++++++++ .../cassandra/guava/GuavaDriverContext.java | 22 +---------- .../cassandra/guava/GuavaSessionBuilder.java | 14 +------ .../server/dao/util/NoSqlAnyDaoNonCloud.java | 27 ++++++++++++++ .../resources/cassandra/schema-keyspace.cql | 21 +++++++++++ .../resources/cassandra/schema-ts-latest.cql | 6 --- .../main/resources/cassandra/schema-ts.cql | 6 --- .../server/dao/NoSqlDaoServiceTestSuite.java | 1 + .../client/tools/migrator/README.md | 2 +- 14 files changed, 143 insertions(+), 45 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/CassandraKeyspaceService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/install/NoSqlKeyspaceService.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDaoNonCloud.java create mode 100644 dao/src/main/resources/cassandra/schema-keyspace.cql diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 2c3251834f..8eec891b85 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService; import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; +import org.thingsboard.server.service.install.NoSqlKeyspaceService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; import org.thingsboard.server.service.install.TsLatestDatabaseSchemaService; @@ -51,6 +52,9 @@ public class ThingsboardInstallService { @Autowired private EntityDatabaseSchemaService entityDatabaseSchemaService; + @Autowired(required = false) + private NoSqlKeyspaceService noSqlKeyspaceService; + @Autowired private TsDatabaseSchemaService tsDatabaseSchemaService; @@ -248,6 +252,10 @@ public class ThingsboardInstallService { log.info("Installing DataBase schema for timeseries..."); + if (noSqlKeyspaceService != null) { + noSqlKeyspaceService.createDatabaseSchema(); + } + tsDatabaseSchemaService.createDatabaseSchema(); if (tsLatestDatabaseSchemaService != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraKeyspaceService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraKeyspaceService.java new file mode 100644 index 0000000000..dfdd05e829 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraKeyspaceService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.NoSqlAnyDaoNonCloud; + +/* +* Create keyspace for Cassandra NoSQL database for non-cloud deployment. +* For cloud service like Astra DBaas admin have to create keyspace manually on cloud UI. +* Then create tokens with database admin role and put it on Thingsboard parameters. +* Without this service cloud DB will end up with exception like +* UnauthorizedException: Missing correct permission on thingsboard +* */ +@Service +@NoSqlAnyDaoNonCloud +@Profile("install") +public class CassandraKeyspaceService extends CassandraAbstractDatabaseSchemaService + implements NoSqlKeyspaceService { + public CassandraKeyspaceService() { + super("schema-keyspace.cql"); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/NoSqlKeyspaceService.java b/application/src/main/java/org/thingsboard/server/service/install/NoSqlKeyspaceService.java new file mode 100644 index 0000000000..5b8d772dca --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/NoSqlKeyspaceService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +public interface NoSqlKeyspaceService extends DatabaseSchemaService { +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 83ff290476..0e8018cebe 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -200,6 +200,14 @@ cassandra: username: "${CASSANDRA_USERNAME:}" # Specify your password password: "${CASSANDRA_PASSWORD:}" + # Astra DB connect https://astra.datastax.com/ + cloud: + # /etc/thingsboard/astra/secure-connect-thingsboard.zip + secure_connect_bundle_path: "${CASSANDRA_CLOUD_SECURE_BUNDLE_PATH:}" + # DucitQPHMzPCBOZqFYexAfKk + client_id: "${CASSANDRA_CLOUD_CLIENT_ID:}" + # ZnF7FpuHp43FP5BzM+KY8wGmSb4Ql6BhT4Z7sOU13ze+gXQ-n7OkFpNuB,oACUIQObQnK0g4bSPoZhK5ejkcF9F.j6f64j71Sr.tiRe0Fsq2hPS1ZCGSfAaIgg63IydG + client_secret: "${CASSANDRA_CLOUD_CLIENT_SECRET:}" # Cassandra cluster connection socket parameters # socket: diff --git a/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java index 255057cc2f..caaf5be6de 100644 --- a/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/transport/TransportNoSqlTestSuite.java @@ -35,6 +35,7 @@ public class TransportNoSqlTestSuite { public static CustomCassandraCQLUnit cassandraUnit = new CustomCassandraCQLUnit( Arrays.asList( + new ClassPathCQLDataSet("cassandra/schema-keyspace.cql", false, false), new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) ), diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java index 6586f42d23..f3f76aba9f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java @@ -24,11 +24,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; import org.springframework.core.env.Profiles; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.dao.cassandra.guava.GuavaSession; import org.thingsboard.server.dao.cassandra.guava.GuavaSessionBuilder; import org.thingsboard.server.dao.cassandra.guava.GuavaSessionUtils; import javax.annotation.PreDestroy; +import java.nio.file.Paths; @Slf4j public abstract class AbstractCassandraCluster { @@ -40,6 +42,13 @@ public abstract class AbstractCassandraCluster { @Value("${cassandra.local_datacenter:datacenter1}") private String localDatacenter; + @Value("${cassandra.cloud.secure_connect_bundle_path:}") + private String cloudSecureConnectBundlePath; + @Value("${cassandra.cloud.client_id:}") + private String cloudClientId; + @Value("${cassandra.cloud.client_secret:}") + private String cloudClientSecret; + @Autowired private CassandraDriverOptions driverOptions; @@ -86,7 +95,14 @@ public abstract class AbstractCassandraCluster { this.sessionBuilder.withKeyspace(this.keyspaceName); } this.sessionBuilder.withLocalDatacenter(localDatacenter); + + if (StringUtils.isNotBlank(cloudSecureConnectBundlePath)) { + this.sessionBuilder.withCloudSecureConnectBundle(Paths.get(cloudSecureConnectBundlePath)); + this.sessionBuilder.withAuthCredentials(cloudClientId, cloudClientSecret); + } + session = sessionBuilder.build(); + if (this.metrics && this.jmx) { MetricRegistry registry = session.getMetrics().orElseThrow( diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaDriverContext.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaDriverContext.java index b5472d78a2..f193b0a7fe 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaDriverContext.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaDriverContext.java @@ -40,26 +40,8 @@ import java.util.function.Predicate; */ public class GuavaDriverContext extends DefaultDriverContext { - public GuavaDriverContext( - DriverConfigLoader configLoader, - List> typeCodecs, - NodeStateListener nodeStateListener, - SchemaChangeListener schemaChangeListener, - RequestTracker requestTracker, - Map localDatacenters, - Map> nodeFilters, - ClassLoader classLoader) { - super( - configLoader, - ProgrammaticArguments.builder() - .addTypeCodecs(typeCodecs.toArray(new TypeCodec[0])) - .withNodeStateListener(nodeStateListener) - .withSchemaChangeListener(schemaChangeListener) - .withRequestTracker(requestTracker) - .withLocalDatacenters(localDatacenters) - .withNodeFilters(nodeFilters) - .withClassLoader(classLoader) - .build()); + public GuavaDriverContext(DriverConfigLoader configLoader, ProgrammaticArguments programmaticArguments) { + super(configLoader, programmaticArguments); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaSessionBuilder.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaSessionBuilder.java index c95c653d64..60ff71a8bd 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaSessionBuilder.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/guava/GuavaSessionBuilder.java @@ -25,18 +25,8 @@ import edu.umd.cs.findbugs.annotations.NonNull; public class GuavaSessionBuilder extends SessionBuilder { @Override - protected DriverContext buildContext( - DriverConfigLoader configLoader, - ProgrammaticArguments programmaticArguments) { - return new GuavaDriverContext( - configLoader, - programmaticArguments.getTypeCodecs(), - programmaticArguments.getNodeStateListener(), - programmaticArguments.getSchemaChangeListener(), - programmaticArguments.getRequestTracker(), - programmaticArguments.getLocalDatacenters(), - programmaticArguments.getNodeFilters(), - programmaticArguments.getClassLoader()); + protected DriverContext buildContext(DriverConfigLoader configLoader, ProgrammaticArguments programmaticArguments) { + return new GuavaDriverContext(configLoader, programmaticArguments); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDaoNonCloud.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDaoNonCloud.java new file mode 100644 index 0000000000..0ec01c8b0a --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDaoNonCloud.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("('${database.ts.type}'=='cassandra' || '${database.ts_latest.type}'=='cassandra') " + + "&& ('${cassandra.cloud.secure_connect_bundle_path}' == null || '${cassandra.cloud.secure_connect_bundle_path}'.isBlank() )") +public @interface NoSqlAnyDaoNonCloud { +} diff --git a/dao/src/main/resources/cassandra/schema-keyspace.cql b/dao/src/main/resources/cassandra/schema-keyspace.cql new file mode 100644 index 0000000000..8323f149ce --- /dev/null +++ b/dao/src/main/resources/cassandra/schema-keyspace.cql @@ -0,0 +1,21 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE KEYSPACE IF NOT EXISTS thingsboard +WITH replication = { + 'class' : 'SimpleStrategy', + 'replication_factor' : 1 +}; diff --git a/dao/src/main/resources/cassandra/schema-ts-latest.cql b/dao/src/main/resources/cassandra/schema-ts-latest.cql index 0de4e88158..10b3edb84e 100644 --- a/dao/src/main/resources/cassandra/schema-ts-latest.cql +++ b/dao/src/main/resources/cassandra/schema-ts-latest.cql @@ -14,12 +14,6 @@ -- limitations under the License. -- -CREATE KEYSPACE IF NOT EXISTS thingsboard -WITH replication = { - 'class' : 'SimpleStrategy', - 'replication_factor' : 1 -}; - CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf ( entity_type text, // (DEVICE, CUSTOMER, TENANT) entity_id timeuuid, diff --git a/dao/src/main/resources/cassandra/schema-ts.cql b/dao/src/main/resources/cassandra/schema-ts.cql index 9cb609add2..6ab099f841 100644 --- a/dao/src/main/resources/cassandra/schema-ts.cql +++ b/dao/src/main/resources/cassandra/schema-ts.cql @@ -14,12 +14,6 @@ -- limitations under the License. -- -CREATE KEYSPACE IF NOT EXISTS thingsboard -WITH replication = { - 'class' : 'SimpleStrategy', - 'replication_factor' : 1 -}; - CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf ( entity_type text, // (DEVICE, CUSTOMER, TENANT) entity_id timeuuid, diff --git a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java index 838be51349..e601218845 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java @@ -33,6 +33,7 @@ public class NoSqlDaoServiceTestSuite { public static CustomCassandraCQLUnit cassandraUnit = new CustomCassandraCQLUnit( Arrays.asList( + new ClassPathCQLDataSet("cassandra/schema-keyspace.cql", false, false), new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) ), diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md b/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md index a468cbd3af..6995632514 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md @@ -63,7 +63,7 @@ Tool execution time depends on DB size, CPU resources and Disk throughput * Note that this this part works only for single node Cassandra Cluster. If you have more nodes - it is better to use `sstableloader` tool. 1. [Optional] install Cassandra on the instance -2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-ts.cql` and `schema-ts-latest.cql` using `source` command +2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-keyspace.cql`, `schema-ts.cql` and `schema-ts-latest.cql` using `source` command 3. Stop Cassandra 4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders 5. Copy generated SSTable files into cassandra data dir using next command: From 4954de20224aa859f303fb78b94d54192f744b25 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 3 Nov 2022 16:48:37 +0200 Subject: [PATCH 02/50] Avoid using 'nulls first' or 'nulls last' for ts_kv --- application/src/main/resources/thingsboard.yml | 2 +- .../AbstractChunkedAggregationTimeseriesDao.java | 5 ++--- .../sqlts/timescale/TimescaleTimeseriesDao.java | 4 +--- .../sqlts/timescale/TsKvTimescaleRepository.java | 15 +++++++-------- .../server/dao/sqlts/ts/TsKvRepository.java | 11 +++++++++-- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 66a603a1a1..75e6b777c8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -531,7 +531,7 @@ spring.servlet.multipart.max-file-size: "50MB" spring.servlet.multipart.max-request-size: "50MB" spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true" -spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" +spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" # Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this default is used # SQL DAO Configuration spring: diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 22c4661fcd..cf726c32c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -20,7 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Sort; +import org.springframework.data.domain.Sort.Direction; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -143,8 +143,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq keyId, query.getStartTs(), query.getEndTs(), - PageRequest.of(0, query.getLimit(), - Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative()))); + PageRequest.ofSize(query.getLimit()).withSort(Direction.fromString(query.getOrder()), "ts")); tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey())); List tsKvEntries = DaoUtil.convertDataList(tsKvEntities); long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 6e9f9e61b8..c9daf1ec6d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -175,9 +175,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements keyId, query.getStartTs(), query.getEndTs(), - PageRequest.of(0, query.getLimit(), - Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative()))); - ; + PageRequest.ofSize(query.getLimit()).withSort(Sort.Direction.fromString(query.getOrder()), "ts")); timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey)); var tsKvEntries = DaoUtil.convertDataList(timescaleTsKvEntities); long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java index 075cffcbc7..821833cad0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java @@ -31,14 +31,13 @@ import java.util.UUID; @TimescaleDBTsOrTsLatestDao public interface TsKvTimescaleRepository extends JpaRepository { - @Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " + - "AND tskv.key = :entityKey " + - "AND tskv.ts >= :startTs AND tskv.ts < :endTs") - List findAllWithLimit( - @Param("entityId") UUID entityId, - @Param("entityKey") int key, - @Param("startTs") long startTs, - @Param("endTs") long endTs, Pageable pageable); + @Query(value = "SELECT * FROM ts_kv WHERE entity_id = :entityId " + + "AND key = :entityKey AND ts >= :startTs AND ts < :endTs", nativeQuery = true) + List findAllWithLimit(@Param("entityId") UUID entityId, + @Param("entityKey") int key, + @Param("startTs") long startTs, + @Param("endTs") long endTs, + Pageable pageable); @Transactional @Modifying diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java index 1d9817a5bd..66d619d0ea 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java @@ -29,8 +29,15 @@ import java.util.UUID; public interface TsKvRepository extends JpaRepository { - @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + - "AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") + /* + * Using native query to avoid adding 'nulls first' or 'nulls last' (ignoring spring.jpa.properties.hibernate.order_by.default_null_ordering) + * to the order so that index scan is done instead of full scan. + * + * Note: even when setting custom NullHandling for the Sort.Order for non-native queries, + * it will be ignored and default_null_ordering will be used + * */ + @Query(value = "SELECT * FROM ts_kv WHERE entity_id = :entityId " + + "AND key = :entityKey AND ts >= :startTs AND ts < :endTs ", nativeQuery = true) List findAllWithLimit(@Param("entityId") UUID entityId, @Param("entityKey") int key, @Param("startTs") long startTs, From d9023d701766d76fe02649fd59a4d6d8d503441e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 3 Nov 2022 17:21:54 +0200 Subject: [PATCH 03/50] Fix comment for hibernate.order_by.default_null_ordering property --- application/src/main/resources/thingsboard.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 75e6b777c8..b9130127ae 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -531,7 +531,8 @@ spring.servlet.multipart.max-file-size: "50MB" spring.servlet.multipart.max-request-size: "50MB" spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true" -spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" # Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this default is used +# Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this parameter is used +spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" # SQL DAO Configuration spring: From 52d7da7afd934d6a21002b4794c5730c21399cf2 Mon Sep 17 00:00:00 2001 From: dlandiak Date: Fri, 4 Nov 2022 10:33:08 +0200 Subject: [PATCH 04/50] inactivity timeout fix - get from attr when persist to telemetry and default found --- .../state/DefaultDeviceStateService.java | 27 +++++++++++- .../state/DefaultDeviceStateServiceTest.java | 41 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index c9c54161dd..38f4ee50c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -24,6 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -83,6 +84,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -152,6 +154,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService attributeOpt = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), SERVER_SCOPE, INACTIVITY_TIMEOUT).get(); + attributeOpt.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> { + if (inactivityTimeout > 0) { + deviceStateData.getState().setInactivityTimeout(inactivityTimeout); + } + }); + } catch (Exception e) { + log.warn("[{}] Failed to fetch inactivity timeout from attribute", deviceStateData.getDeviceId(), e); + } + } } private EntityKeyType getKeyType() { diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index d5d74f2512..bc0e067442 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -15,27 +15,45 @@ */ package org.thingsboard.server.service.state; +import com.google.common.util.concurrent.Futures; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; +import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; +import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_TIMEOUT; @RunWith(MockitoJUnitRunner.class) public class DefaultDeviceStateServiceTest { @@ -83,4 +101,25 @@ public class DefaultDeviceStateServiceTest { Mockito.verify(service, times(1)).fetchDeviceStateDataUsingEntityDataQuery(deviceId); } + @Test + public void givenPersistToTelemetryAndDefaultInactivityTimeoutFetched_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() { + var defaultInactivityTimeoutInSec = 60L; + service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec); + service.setPersistToTelemetry(true); + + var deviceUuid = UUID.randomUUID(); + var deviceId = new DeviceId(deviceUuid); + + when(attributesService.find(any(), any(), anyString(), anyString())) + .thenReturn(Futures.immediateFuture(Optional.of(new BaseAttributeKvEntry(0, new LongDataEntry(INACTIVITY_TIMEOUT, 5000L))))); + + var latest = + Map.of(EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000)))); + DeviceStateData deviceStateData = service.toDeviceStateData(new EntityData(deviceId, latest, Map.of()), new DeviceIdInfo(TenantId.SYS_TENANT_ID.getId(), UUID.randomUUID(), deviceUuid)); + + Mockito.verify(attributesService, times(1)).find(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, INACTIVITY_TIMEOUT); + + Assert.assertEquals(5000L, deviceStateData.getState().getInactivityTimeout()); + } + } \ No newline at end of file From 9b54f009e6a9c6e2b01714001c9b39a009ef7c20 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 7 Nov 2022 11:29:27 +0100 Subject: [PATCH 05/50] upgrade asset-profiles using ExecutorService --- .../install/DbUpgradeExecutorService.java | 26 ++++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 30 +++++++++++-------- 2 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java diff --git a/application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java b/application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java new file mode 100644 index 0000000000..b3ce77ac7a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/DbUpgradeExecutorService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; +import org.thingsboard.server.service.executors.DbCallbackExecutorService; + +@Component +@Profile("install") +public class DbUpgradeExecutorService extends DbCallbackExecutorService { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 092726d0c8..668f20ce63 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.install; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -33,13 +35,11 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.rule.RuleChainService; -import org.thingsboard.server.dao.tenant.TenantProfileService; +import org.thingsboard.server.dao.sql.asset.AssetRepository; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -56,7 +56,10 @@ import java.sql.SQLException; import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; @@ -110,7 +113,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService private DeviceService deviceService; @Autowired - private AssetService assetService; + private AssetRepository assetRepository; @Autowired private DeviceProfileService deviceProfileService; @@ -129,10 +132,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService private TbRuleEngineQueueConfigService queueConfig; @Autowired - private RuleChainService ruleChainService; - - @Autowired - private TenantProfileService tenantProfileService; + private DbUpgradeExecutorService dbUpgradeExecutor; @Override public void upgradeDatabase(String fromVersion) throws Exception { @@ -623,17 +623,20 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.info("Creating default asset profiles..."); PageLink pageLink = new PageLink(100); PageData pageData; + List> futures = new ArrayList<>(); do { pageData = tenantService.findTenants(pageLink); for (Tenant tenant : pageData.getData()) { - List assetTypes = assetService.findAssetTypesByTenantId(tenant.getId()).get(); + Set assetTypes = new HashSet<>(assetRepository.findTenantAssetTypes(tenant.getUuidId())); + assetTypes.remove("default"); + try { - assetProfileService.createDefaultAssetProfile(tenant.getId()); + futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.createDefaultAssetProfile(tenant.getId()))); } catch (Exception e) { } - for (EntitySubtype assetType : assetTypes) { + for (String assetType : assetTypes) { try { - assetProfileService.findOrCreateAssetProfile(tenant.getId(), assetType.getType()); + futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.findOrCreateAssetProfile(tenant.getId(), assetType))); } catch (Exception e) { } } @@ -641,6 +644,8 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService pageLink = pageLink.nextPageLink(); } while (pageData.hasNext()); + Futures.allAsList(futures).get(); + log.info("Updating asset profiles..."); conn.createStatement().execute("call update_asset_profiles()"); @@ -728,5 +733,4 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService return queue; } - } From a9dabe14ca191e2d0a92adf210d70d8bca409555 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 7 Nov 2022 11:51:09 +0100 Subject: [PATCH 06/50] refactored --- .../server/service/install/SqlDatabaseUpgradeService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 668f20ce63..4b6f1e1a52 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -57,9 +57,7 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; @@ -627,7 +625,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService do { pageData = tenantService.findTenants(pageLink); for (Tenant tenant : pageData.getData()) { - Set assetTypes = new HashSet<>(assetRepository.findTenantAssetTypes(tenant.getUuidId())); + List assetTypes = assetRepository.findTenantAssetTypes(tenant.getUuidId()); assetTypes.remove("default"); try { From 11ee41bf341dd5500b2b91bd2928ce34cd44893f Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 7 Nov 2022 14:42:49 +0200 Subject: [PATCH 07/50] Adding partition to edge_event table --- .../main/data/upgrade/3.4.1/schema_update.sql | 8 +- .../main/data/upgrade/3.4.2/schema_update.sql | 76 +++++++++++++++++ .../install/ThingsboardInstallService.java | 7 +- .../install/SqlDatabaseUpgradeService.java | 12 +++ .../update/DefaultDataUpdateService.java | 22 +++++ .../service/ttl/EdgeEventsCleanUpService.java | 25 +++++- .../src/main/resources/thingsboard.yml | 1 + .../BaseAuditLogControllerTest.java | 1 - .../BaseEdgeEventControllerTest.java | 69 +++++++++++++++ .../resources/application-test.properties | 5 +- .../server/dao/edge/BaseEdgeEventService.java | 12 +-- .../server/dao/edge/EdgeEventDao.java | 2 + .../dao/sql/edge/JpaBaseEdgeEventDao.java | 85 ++++++++++++------- .../resources/sql/schema-entities-idx.sql | 2 + .../main/resources/sql/schema-entities.sql | 4 +- 15 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 application/src/main/data/upgrade/3.4.2/schema_update.sql diff --git a/application/src/main/data/upgrade/3.4.1/schema_update.sql b/application/src/main/data/upgrade/3.4.1/schema_update.sql index 5891246b44..66169f0120 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update.sql @@ -55,16 +55,16 @@ CREATE OR REPLACE PROCEDURE migrate_audit_logs(IN start_time_ms BIGINT, IN end_t LANGUAGE plpgsql AS $$ DECLARE - p RECORD; + prt RECORD; partition_end_ts BIGINT; BEGIN FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_audit_log WHERE created_time >= start_time_ms AND created_time < end_time_ms LOOP - partition_end_ts = p.partition_ts + partition_size_ms; - RAISE NOTICE '[audit_log] Partition to create : [%-%]', p.partition_ts, partition_end_ts; + partition_end_ts = prt.partition_ts + partition_size_ms; + RAISE NOTICE '[audit_log] Partition to create : [%-%]', prt.partition_ts, partition_end_ts; EXECUTE format('CREATE TABLE IF NOT EXISTS audit_log_%s PARTITION OF audit_log ' || - 'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts); + 'FOR VALUES FROM ( %s ) TO ( %s )', prt.partition_ts, prt.partition_ts, partition_end_ts); END LOOP; INSERT INTO audit_log diff --git a/application/src/main/data/upgrade/3.4.2/schema_update.sql b/application/src/main/data/upgrade/3.4.2/schema_update.sql new file mode 100644 index 0000000000..10c009b740 --- /dev/null +++ b/application/src/main/data/upgrade/3.4.2/schema_update.sql @@ -0,0 +1,76 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +DO +$$ + DECLARE table_partition RECORD; + BEGIN + -- in case of running the upgrade script a second time: + IF NOT (SELECT exists(SELECT FROM pg_tables WHERE tablename = 'old_edge_event')) THEN + ALTER TABLE edge_event RENAME TO old_edge_event; + ALTER INDEX IF EXISTS idx_edge_event_tenant_id_and_created_time RENAME TO idx_old_edge_event_tenant_id_and_created_time; + + FOR table_partition IN SELECT tablename AS name, split_part(tablename, '_', 3) AS partition_ts + FROM pg_tables WHERE tablename LIKE 'edge_event_%' + LOOP + EXECUTE format('ALTER TABLE %s RENAME TO old_edge_event_%s', table_partition.name, table_partition.partition_ts); + END LOOP; + ELSE + RAISE NOTICE 'Table old_edge_event already exists, leaving as is'; + END IF; + END; +$$; + + +CREATE TABLE IF NOT EXISTS edge_event ( + id uuid NOT NULL, + created_time bigint NOT NULL, + edge_id uuid, + edge_event_type varchar(255), + edge_event_uid varchar(255), + entity_id uuid, + edge_event_action varchar(255), + body varchar(10000000), + tenant_id uuid, + ts bigint NOT NULL + ) PARTITION BY RANGE (created_time); +CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_and_created_time ON edge_event(tenant_id, created_time DESC); + + +CREATE OR REPLACE PROCEDURE migrate_edge_event(IN start_time_ms BIGINT, IN end_time_ms BIGINT, IN partition_size_ms BIGINT) + LANGUAGE plpgsql AS +$$ +DECLARE + p RECORD; + partition_end_ts BIGINT; +BEGIN + FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_edge_event + WHERE created_time >= start_time_ms AND created_time < end_time_ms + LOOP + partition_end_ts = p.partition_ts + partition_size_ms; + RAISE NOTICE '[edge_event] Partition to create : [%-%]', p.partition_ts, partition_end_ts; + EXECUTE format('CREATE TABLE IF NOT EXISTS edge_event_%s PARTITION OF edge_event ' || + 'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts); + END LOOP; + + INSERT INTO edge_event + SELECT id, created_time, edge_id, edge_event_type, edge_event_uid, entity_id, edge_event_action, body, tenant_id, ts + FROM old_edge_event + WHERE created_time >= start_time_ms AND created_time < end_time_ms; +END; +$$; + + diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 33a6d4bdd1..5fd8c47997 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -230,9 +230,14 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.4.1"); dataUpdateService.updateData("3.4.1"); log.info("Updating system data..."); + break; + case "3.4.2": + log.info("Upgrading ThingsBoard from version 3.4.2 to 3.5.0 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.4.2"); + dataUpdateService.updateData("3.4.2"); + log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; - //TODO update CacheCleanupService on the next version upgrade default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 092726d0c8..c7ccf4cd18 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -654,6 +654,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.4.2": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.2", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3005000;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index a8cf5374a2..68c1c50140 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -64,6 +64,7 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.audit.AuditLogDao; +import org.thingsboard.server.dao.edge.EdgeEventDao; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.event.EventService; @@ -142,6 +143,9 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private AuditLogDao auditLogDao; + @Autowired + private EdgeEventDao edgeEventDao; + @Override public void updateData(String fromVersion) throws Exception { switch (fromVersion) { @@ -189,6 +193,24 @@ public class DefaultDataUpdateService implements DataUpdateService { } else { log.info("Skipping audit logs migration"); } + boolean skipEdgeEventsMigrationTemp = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); + if (!skipEdgeEventsMigrationTemp) { + log.info("Updating data from version 3.4.1 to 3.4.2 ..."); + log.info("Starting edge events migration. Can be skipped with TB_SKIP_EDGE_EVENTS_MIGRATION env variable set to true"); + edgeEventDao.migrateEdgeEvents(); + } else { + log.info("Skipping edge events migration"); + } + break; + case "3.5.0": + boolean skipEdgeEventsMigration = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); + if (!skipEdgeEventsMigration) { + log.info("Updating data from version 3.4.2 to 3.5.0 ..."); + log.info("Starting edge events migration. Can be skipped with TB_SKIP_EDGE_EVENTS_MIGRATION env variable set to true"); + edgeEventDao.migrateEdgeEvents(); + } else { + log.info("Skipping edge events migration"); + } break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java index c4ce4e49b2..880d3eb125 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java @@ -17,15 +17,23 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.edge.EdgeEventDao; import org.thingsboard.server.dao.edge.EdgeEventService; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; +import java.util.concurrent.TimeUnit; + +import static org.thingsboard.server.dao.model.ModelConstants.EDGE_EVENT_COLUMN_FAMILY_NAME; + @TbCoreComponent @Slf4j @Service +@ConditionalOnExpression("${sql.ttl.edge_events.enabled:true} && ${sql.ttl.edge_events.edge_event_ttl:0} > 0") public class EdgeEventsCleanUpService extends AbstractCleanUpService { public static final String RANDOM_DELAY_INTERVAL_MS_EXPRESSION = @@ -34,20 +42,29 @@ public class EdgeEventsCleanUpService extends AbstractCleanUpService { @Value("${sql.ttl.edge_events.edge_events_ttl}") private long ttl; - @Value("${sql.ttl.edge_events.enabled}") + @Value("${sql.edge_events.partition_size:168}") + private int partitionSizeInHours; + + @Value("${sql.ttl.edge_events.enabled:true}") private boolean ttlTaskExecutionEnabled; private final EdgeEventService edgeEventService; - public EdgeEventsCleanUpService(PartitionService partitionService, EdgeEventService edgeEventService) { + private final SqlPartitioningRepository partitioningRepository; + + public EdgeEventsCleanUpService(PartitionService partitionService, EdgeEventService edgeEventService, SqlPartitioningRepository partitioningRepository) { super(partitionService); this.edgeEventService = edgeEventService; + this.partitioningRepository = partitioningRepository; } @Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}") public void cleanUp() { - if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { - edgeEventService.cleanupEvents(ttl); + long edgeEventsExpTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttl); + if(isSystemTenantPartitionMine()) { + edgeEventService.cleanupEvents(edgeEventsExpTime); + } else { + partitioningRepository.cleanupPartitionsCache(EDGE_EVENT_COLUMN_FAMILY_NAME, edgeEventsExpTime, TimeUnit.HOURS.toMillis(partitionSizeInHours)); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 66a603a1a1..2a8ea20611 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -265,6 +265,7 @@ sql: batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}" batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}" stats_print_interval_ms: "${SQL_EDGE_EVENTS_BATCH_STATS_PRINT_MS:10000}" + partition_size: "${SQL_EDGE_EVENTS_PARTITION_SIZE_HOURS:168}" # Number of hours to partition the events. The current value corresponds to one week. audit_logs: partition_size: "${SQL_AUDIT_LOGS_PARTITION_SIZE_HOURS:168}" # Default value - 1 week # Specify whether to sort entities before batch update. Should be enabled for cluster mode to avoid deadlocks diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java index c46f7d9c46..4f8d07864b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java @@ -178,7 +178,6 @@ public abstract class BaseAuditLogControllerTest extends AbstractControllerTest reset(partitioningRepository); AuditLog auditLog = createAuditLog(ActionType.LOGIN, tenantAdminUserId); verify(partitioningRepository).createPartitionIfNotExists(eq("audit_log"), eq(auditLog.getCreatedTime()), eq(partitionDurationInMs)); - List partitions = partitioningRepository.fetchPartitions("audit_log"); assertThat(partitions).singleElement().satisfies(partitionStartTs -> { assertThat(partitionStartTs).isEqualTo(partitioningRepository.calculatePartitionStartTime(auditLog.getCreatedTime(), partitionDurationInMs)); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEdgeEventControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEdgeEventControllerTest.java index ef66270130..ccf218c132 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEdgeEventControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEdgeEventControllerTest.java @@ -22,6 +22,9 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; @@ -29,16 +32,27 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.edge.EdgeEventDao; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.service.ttl.EdgeEventsCleanUpService; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @TestPropertySource(properties = { @@ -50,6 +64,18 @@ public abstract class BaseEdgeEventControllerTest extends AbstractControllerTest private Tenant savedTenant; private User tenantAdmin; + @Autowired + private EdgeEventDao edgeEventDao; + @SpyBean + private SqlPartitioningRepository partitioningRepository; + @Autowired + private EdgeEventsCleanUpService edgeEventsCleanUpService; + + @Value("#{${sql.edge_events.partition_size} * 60 * 60 * 1000}") + private long partitionDurationInMs; + @Value("${sql.ttl.edge_events.edge_event_ttl}") + private long edgeEventTtlInSec; + @Before public void beforeTest() throws Exception { loginSysAdmin(); @@ -114,6 +140,34 @@ public abstract class BaseEdgeEventControllerTest extends AbstractControllerTest Assert.assertTrue(edgeEvents.stream().anyMatch(ee -> EdgeEventType.RELATION.equals(ee.getType()))); } + @Test + public void saveEdgeEvent_thenCreatePartitionIfNotExist() { + reset(partitioningRepository); + EdgeEvent edgeEvent = createEdgeEvent(); + verify(partitioningRepository).createPartitionIfNotExists(eq("edge_event"), eq(edgeEvent.getCreatedTime()), eq(partitionDurationInMs)); + List partitions = partitioningRepository.fetchPartitions("edge_event"); + assertThat(partitions).singleElement().satisfies(partitionStartTs -> { + assertThat(partitionStartTs).isEqualTo(partitioningRepository.calculatePartitionStartTime(edgeEvent.getCreatedTime(), partitionDurationInMs)); + }); + } + + @Test + public void cleanUpEdgeEventByTtl_dropOldPartitions() { + long oldEdgeEventTs = LocalDate.of(2020, 10, 1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli(); + long partitionStartTs = partitioningRepository.calculatePartitionStartTime(oldEdgeEventTs, partitionDurationInMs); + partitioningRepository.createPartitionIfNotExists("edge_event", oldEdgeEventTs, partitionDurationInMs); + List partitions = partitioningRepository.fetchPartitions("edge_event"); + assertThat(partitions).contains(partitionStartTs); + + edgeEventsCleanUpService.cleanUp(); + partitions = partitioningRepository.fetchPartitions("edge_event"); + assertThat(partitions).doesNotContain(partitionStartTs); + assertThat(partitions).allSatisfy(partitionsStart -> { + long partitionEndTs = partitionsStart + partitionDurationInMs; + assertThat(partitionEndTs).isGreaterThan(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(edgeEventTtlInSec)); + }); + } + private List findEdgeEvents(EdgeId edgeId) throws Exception { return doGetTypedWithTimePageLink("/api/edge/" + edgeId.toString() + "/events?", new TypeReference>() { @@ -134,4 +188,19 @@ public abstract class BaseEdgeEventControllerTest extends AbstractControllerTest return asset; } + private EdgeEvent createEdgeEvent() { + EdgeEvent edgeEvent = new EdgeEvent(); + edgeEvent.setCreatedTime(System.currentTimeMillis()); + edgeEvent.setTenantId(tenantId); + edgeEvent.setAction(EdgeEventActionType.ADDED); + edgeEvent.setEntityId(tenantAdmin.getUuidId()); + edgeEvent.setType(EdgeEventType.ALARM); + try { + edgeEventDao.saveAsync(edgeEvent).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + return edgeEvent; + } + } diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index eacd1733d3..fb2fdfe0a4 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -59,4 +59,7 @@ queue.rule-engine.queues[2].processing-strategy.max-pause-between-retries=0 usage.stats.report.enabled=false sql.audit_logs.partition_size=24 -sql.ttl.audit_logs.ttl=2592000 \ No newline at end of file +sql.ttl.audit_logs.ttl=2592000 + +sql.edge_events.partition_size=168 +sql.ttl.edge_events.edge_event_ttl=2592000 \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java index f9e94af613..09322aa555 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.edge; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; @@ -30,11 +29,14 @@ import org.thingsboard.server.dao.service.DataValidator; @Slf4j public class BaseEdgeEventService implements EdgeEventService { - @Autowired - private EdgeEventDao edgeEventDao; + private final EdgeEventDao edgeEventDao; - @Autowired - private DataValidator edgeEventValidator; + private final DataValidator edgeEventValidator; + + public BaseEdgeEventService(EdgeEventDao edgeEventDao, DataValidator edgeEventValidator) { + this.edgeEventDao = edgeEventDao; + this.edgeEventValidator = edgeEventValidator; + } @Override public ListenableFuture saveAsync(EdgeEvent edgeEvent) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java index 7a43d6c065..cb31869213 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeEventDao.java @@ -54,4 +54,6 @@ public interface EdgeEventDao extends Dao { */ void cleanupEvents(long ttl); + void migrateEdgeEvents(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index df182659bf..334d5a5671 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -17,10 +17,11 @@ package org.thingsboard.server.dao.sql.edge; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -31,19 +32,17 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.edge.EdgeEventDao; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.EdgeEventEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import org.thingsboard.server.dao.util.SqlDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.Comparator; import java.util.Objects; import java.util.UUID; @@ -51,19 +50,25 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; - -@Slf4j @Component @SqlDao +@RequiredArgsConstructor +@Slf4j public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao implements EdgeEventDao { private final UUID systemTenantId = NULL_UUID; - @Autowired - ScheduledLogExecutorComponent logExecutor; + private final ScheduledLogExecutorComponent logExecutor; + + private final StatsFactory statsFactory; + + private final EdgeEventRepository edgeEventRepository; - @Autowired - private StatsFactory statsFactory; + private final EdgeEventInsertRepository edgeEventInsertRepository; + + private final SqlPartitioningRepository partitioningRepository; + + private final JdbcTemplate jdbcTemplate; @Value("${sql.edge_events.batch_size:1000}") private int batchSize; @@ -74,13 +79,14 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao queue; + @Value("${sql.edge_events.partitions_size:168}") + private int partitionSizeInHours; - @Autowired - private EdgeEventRepository edgeEventRepository; + @Value("${sql.ttl.edge_events.edge_events_ttl:2628000}") + private long edge_events_ttl; - @Autowired - private EdgeEventInsertRepository edgeEventInsertRepository; + private static final String TABLE_NAME = ModelConstants.EDGE_EVENT_COLUMN_FAMILY_NAME; + private TbSqlBlockingQueueWrapper queue; @Override protected Class getEntityClass() { @@ -140,6 +146,7 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao 0 ? System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(edge_events_ttl) : 1480982400000L; + + long currentTime = System.currentTimeMillis(); + var partitionStepInMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); + long numberOfPartitions = (currentTime - startTime) / partitionStepInMs; + + if (numberOfPartitions > 1000) { + String error = "Please adjust your edge event partitioning configuration. Configuration with partition size " + + "of " + partitionSizeInHours + " hours and corresponding TTL will use " + numberOfPartitions + " " + + "(> 1000) partitions which is not recommended!"; + log.error(error); + throw new RuntimeException(error); + } + + while (startTime < currentTime) { + var endTime = startTime + partitionStepInMs; + log.info("Migrating edge event for time period: {} - {}", startTime, endTime); + callMigrationFunction(startTime, endTime, partitionStepInMs); + startTime = endTime; } + log.info("Event edge migration finished"); + + jdbcTemplate.execute("DROP TABLE IF EXISTS old_edge_event"); + } + + private void callMigrationFunction(long startTime, long endTime, long partitionSIzeInMs) { + jdbcTemplate.update("CALL migrate_edge_event(?, ?, ?)", startTime, endTime, partitionSIzeInMs); } + } diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index 34862e5af3..e7586b17a8 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -50,6 +50,8 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time DESC); +CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_and_created_time ON edge_event(tenant_id, created_time DESC); + CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 51df863ae5..73039274a6 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -719,7 +719,7 @@ CREATE TABLE IF NOT EXISTS edge ( ); CREATE TABLE IF NOT EXISTS edge_event ( - id uuid NOT NULL CONSTRAINT edge_event_pkey PRIMARY KEY, + id uuid NOT NULL, created_time bigint NOT NULL, edge_id uuid, edge_event_type varchar(255), @@ -729,7 +729,7 @@ CREATE TABLE IF NOT EXISTS edge_event ( body varchar(10000000), tenant_id uuid, ts bigint NOT NULL -); +) PARTITION BY RANGE(created_time); CREATE TABLE IF NOT EXISTS rpc ( id uuid NOT NULL CONSTRAINT rpc_pkey PRIMARY KEY, From 5b3eb26a3bed60bd074e4d508cca406f827cd737 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 7 Nov 2022 14:45:46 +0200 Subject: [PATCH 08/50] Remove unnecessary changes in audit-log schema_update.sql --- application/src/main/data/upgrade/3.4.1/schema_update.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/data/upgrade/3.4.1/schema_update.sql b/application/src/main/data/upgrade/3.4.1/schema_update.sql index 66169f0120..5891246b44 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update.sql @@ -55,16 +55,16 @@ CREATE OR REPLACE PROCEDURE migrate_audit_logs(IN start_time_ms BIGINT, IN end_t LANGUAGE plpgsql AS $$ DECLARE - prt RECORD; + p RECORD; partition_end_ts BIGINT; BEGIN FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_audit_log WHERE created_time >= start_time_ms AND created_time < end_time_ms LOOP - partition_end_ts = prt.partition_ts + partition_size_ms; - RAISE NOTICE '[audit_log] Partition to create : [%-%]', prt.partition_ts, partition_end_ts; + partition_end_ts = p.partition_ts + partition_size_ms; + RAISE NOTICE '[audit_log] Partition to create : [%-%]', p.partition_ts, partition_end_ts; EXECUTE format('CREATE TABLE IF NOT EXISTS audit_log_%s PARTITION OF audit_log ' || - 'FOR VALUES FROM ( %s ) TO ( %s )', prt.partition_ts, prt.partition_ts, partition_end_ts); + 'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts); END LOOP; INSERT INTO audit_log From a098e33d8dfcc15196795a53b8de9e634c2b15b4 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 7 Nov 2022 17:32:04 +0200 Subject: [PATCH 09/50] Set up to 3.4.2 version instead of 3.5.0 --- .../main/data/upgrade/3.4.1/schema_update.sql | 60 +++++++++++++++ .../main/data/upgrade/3.4.2/schema_update.sql | 76 ------------------- .../install/ThingsboardInstallService.java | 6 -- .../install/SqlDatabaseUpgradeService.java | 12 --- .../sql/edge/EdgeEventInsertRepository.java | 2 + 5 files changed, 62 insertions(+), 94 deletions(-) delete mode 100644 application/src/main/data/upgrade/3.4.2/schema_update.sql diff --git a/application/src/main/data/upgrade/3.4.1/schema_update.sql b/application/src/main/data/upgrade/3.4.1/schema_update.sql index 5891246b44..dd9eadcbea 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update.sql @@ -73,3 +73,63 @@ BEGIN WHERE created_time >= start_time_ms AND created_time < end_time_ms; END; $$; + + +DO +$$ + DECLARE table_partition RECORD; + BEGIN + -- in case of running the upgrade script a second time: + IF NOT (SELECT exists(SELECT FROM pg_tables WHERE tablename = 'old_edge_event')) THEN + ALTER TABLE edge_event RENAME TO old_edge_event; + ALTER INDEX IF EXISTS idx_edge_event_tenant_id_and_created_time RENAME TO idx_old_edge_event_tenant_id_and_created_time; + + FOR table_partition IN SELECT tablename AS name, split_part(tablename, '_', 3) AS partition_ts + FROM pg_tables WHERE tablename LIKE 'edge_event_%' + LOOP + EXECUTE format('ALTER TABLE %s RENAME TO old_edge_event_%s', table_partition.name, table_partition.partition_ts); + END LOOP; + ELSE + RAISE NOTICE 'Table old_edge_event already exists, leaving as is'; + END IF; +END; +$$; + + +CREATE TABLE IF NOT EXISTS edge_event ( + id uuid NOT NULL, + created_time bigint NOT NULL, + edge_id uuid, + edge_event_type varchar(255), + edge_event_uid varchar(255), + entity_id uuid, + edge_event_action varchar(255), + body varchar(10000000), + tenant_id uuid, + ts bigint NOT NULL + ) PARTITION BY RANGE (created_time); +CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_and_created_time ON edge_event(tenant_id, created_time DESC); + + +CREATE OR REPLACE PROCEDURE migrate_edge_event(IN start_time_ms BIGINT, IN end_time_ms BIGINT, IN partition_size_ms BIGINT) + LANGUAGE plpgsql AS +$$ +DECLARE + p RECORD; + partition_end_ts BIGINT; +BEGIN + FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_edge_event + WHERE created_time >= start_time_ms AND created_time < end_time_ms + LOOP + partition_end_ts = p.partition_ts + partition_size_ms; + RAISE NOTICE '[edge_event] Partition to create : [%-%]', p.partition_ts, partition_end_ts; + EXECUTE format('CREATE TABLE IF NOT EXISTS edge_event_%s PARTITION OF edge_event ' || + 'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts); + END LOOP; + + INSERT INTO edge_event + SELECT id, created_time, edge_id, edge_event_type, edge_event_uid, entity_id, edge_event_action, body, tenant_id, ts + FROM old_edge_event + WHERE created_time >= start_time_ms AND created_time < end_time_ms; +END; +$$; \ No newline at end of file diff --git a/application/src/main/data/upgrade/3.4.2/schema_update.sql b/application/src/main/data/upgrade/3.4.2/schema_update.sql deleted file mode 100644 index 10c009b740..0000000000 --- a/application/src/main/data/upgrade/3.4.2/schema_update.sql +++ /dev/null @@ -1,76 +0,0 @@ --- --- Copyright © 2016-2022 The Thingsboard Authors --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -DO -$$ - DECLARE table_partition RECORD; - BEGIN - -- in case of running the upgrade script a second time: - IF NOT (SELECT exists(SELECT FROM pg_tables WHERE tablename = 'old_edge_event')) THEN - ALTER TABLE edge_event RENAME TO old_edge_event; - ALTER INDEX IF EXISTS idx_edge_event_tenant_id_and_created_time RENAME TO idx_old_edge_event_tenant_id_and_created_time; - - FOR table_partition IN SELECT tablename AS name, split_part(tablename, '_', 3) AS partition_ts - FROM pg_tables WHERE tablename LIKE 'edge_event_%' - LOOP - EXECUTE format('ALTER TABLE %s RENAME TO old_edge_event_%s', table_partition.name, table_partition.partition_ts); - END LOOP; - ELSE - RAISE NOTICE 'Table old_edge_event already exists, leaving as is'; - END IF; - END; -$$; - - -CREATE TABLE IF NOT EXISTS edge_event ( - id uuid NOT NULL, - created_time bigint NOT NULL, - edge_id uuid, - edge_event_type varchar(255), - edge_event_uid varchar(255), - entity_id uuid, - edge_event_action varchar(255), - body varchar(10000000), - tenant_id uuid, - ts bigint NOT NULL - ) PARTITION BY RANGE (created_time); -CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_and_created_time ON edge_event(tenant_id, created_time DESC); - - -CREATE OR REPLACE PROCEDURE migrate_edge_event(IN start_time_ms BIGINT, IN end_time_ms BIGINT, IN partition_size_ms BIGINT) - LANGUAGE plpgsql AS -$$ -DECLARE - p RECORD; - partition_end_ts BIGINT; -BEGIN - FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_edge_event - WHERE created_time >= start_time_ms AND created_time < end_time_ms - LOOP - partition_end_ts = p.partition_ts + partition_size_ms; - RAISE NOTICE '[edge_event] Partition to create : [%-%]', p.partition_ts, partition_end_ts; - EXECUTE format('CREATE TABLE IF NOT EXISTS edge_event_%s PARTITION OF edge_event ' || - 'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts); - END LOOP; - - INSERT INTO edge_event - SELECT id, created_time, edge_id, edge_event_type, edge_event_uid, entity_id, edge_event_action, body, tenant_id, ts - FROM old_edge_event - WHERE created_time >= start_time_ms AND created_time < end_time_ms; -END; -$$; - - diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 5fd8c47997..39953ef014 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -230,12 +230,6 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.4.1"); dataUpdateService.updateData("3.4.1"); log.info("Updating system data..."); - break; - case "3.4.2": - log.info("Upgrading ThingsBoard from version 3.4.2 to 3.5.0 ..."); - databaseEntitiesUpgradeService.upgradeDatabase("3.4.2"); - dataUpdateService.updateData("3.4.2"); - log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; //TODO update CacheCleanupService on the next version upgrade diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index c7ccf4cd18..092726d0c8 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -654,18 +654,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; - case "3.4.2": - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - log.info("Updating schema ..."); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.2", SCHEMA_UPDATE_SQL); - loadSql(schemaUpdateFile, conn); - log.info("Updating schema settings..."); - conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3005000;"); - log.info("Schema updated."); - } catch (Exception e) { - log.error("Failed updating schema!!!", e); - } - break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java index 694d68556f..ec7a6ccfab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java @@ -51,9 +51,11 @@ public class EdgeEventInsertRepository { protected void doInTransactionWithoutResult(TransactionStatus status) { jdbcTemplate.batchUpdate(INSERT, new BatchPreparedStatementSetter() { @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { EdgeEventEntity edgeEvent = entities.get(i); ps.setObject(1, edgeEvent.getId()); + ps.setLong(2, edgeEvent.getCreatedTime()); ps.setObject(3, edgeEvent.getEdgeId()); ps.setString(4, edgeEvent.getEdgeEventType().name()); From d0090f8b7a195de25416f39cef059e4d84fb334f Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 7 Nov 2022 17:52:44 +0200 Subject: [PATCH 10/50] Removed unnecessary unused case for DataUpdateService --- .../server/install/ThingsboardInstallService.java | 1 + .../install/update/DefaultDataUpdateService.java | 10 ---------- .../server/controller/BaseAuditLogControllerTest.java | 1 + .../server/dao/sql/edge/EdgeEventInsertRepository.java | 2 -- .../server/dao/sql/edge/JpaBaseEdgeEventDao.java | 1 + 5 files changed, 3 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 39953ef014..33a6d4bdd1 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -232,6 +232,7 @@ public class ThingsboardInstallService { log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; + //TODO update CacheCleanupService on the next version upgrade default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 68c1c50140..ea6d278844 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -202,16 +202,6 @@ public class DefaultDataUpdateService implements DataUpdateService { log.info("Skipping edge events migration"); } break; - case "3.5.0": - boolean skipEdgeEventsMigration = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); - if (!skipEdgeEventsMigration) { - log.info("Updating data from version 3.4.2 to 3.5.0 ..."); - log.info("Starting edge events migration. Can be skipped with TB_SKIP_EDGE_EVENTS_MIGRATION env variable set to true"); - edgeEventDao.migrateEdgeEvents(); - } else { - log.info("Skipping edge events migration"); - } - break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java index 4f8d07864b..c46f7d9c46 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAuditLogControllerTest.java @@ -178,6 +178,7 @@ public abstract class BaseAuditLogControllerTest extends AbstractControllerTest reset(partitioningRepository); AuditLog auditLog = createAuditLog(ActionType.LOGIN, tenantAdminUserId); verify(partitioningRepository).createPartitionIfNotExists(eq("audit_log"), eq(auditLog.getCreatedTime()), eq(partitionDurationInMs)); + List partitions = partitioningRepository.fetchPartitions("audit_log"); assertThat(partitions).singleElement().satisfies(partitionStartTs -> { assertThat(partitionStartTs).isEqualTo(partitioningRepository.calculatePartitionStartTime(auditLog.getCreatedTime(), partitionDurationInMs)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java index ec7a6ccfab..694d68556f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeEventInsertRepository.java @@ -51,11 +51,9 @@ public class EdgeEventInsertRepository { protected void doInTransactionWithoutResult(TransactionStatus status) { jdbcTemplate.batchUpdate(INSERT, new BatchPreparedStatementSetter() { @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { EdgeEventEntity edgeEvent = entities.get(i); ps.setObject(1, edgeEvent.getId()); - ps.setLong(2, edgeEvent.getCreatedTime()); ps.setObject(3, edgeEvent.getEdgeId()); ps.setString(4, edgeEvent.getEdgeEventType().name()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index 334d5a5671..a0efcde00e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -86,6 +86,7 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao queue; @Override From fba0cd031b8f4cbd25c5be853b0dab3b152924f0 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 7 Nov 2022 18:13:16 +0200 Subject: [PATCH 11/50] Refactoring code and optimize import --- application/src/main/data/upgrade/3.4.1/schema_update.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/data/upgrade/3.4.1/schema_update.sql b/application/src/main/data/upgrade/3.4.1/schema_update.sql index dd9eadcbea..0f4354c0ca 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update.sql @@ -95,7 +95,6 @@ $$ END; $$; - CREATE TABLE IF NOT EXISTS edge_event ( id uuid NOT NULL, created_time bigint NOT NULL, @@ -110,7 +109,6 @@ CREATE TABLE IF NOT EXISTS edge_event ( ) PARTITION BY RANGE (created_time); CREATE INDEX IF NOT EXISTS idx_edge_event_tenant_id_and_created_time ON edge_event(tenant_id, created_time DESC); - CREATE OR REPLACE PROCEDURE migrate_edge_event(IN start_time_ms BIGINT, IN end_time_ms BIGINT, IN partition_size_ms BIGINT) LANGUAGE plpgsql AS $$ @@ -132,4 +130,4 @@ BEGIN FROM old_edge_event WHERE created_time >= start_time_ms AND created_time < end_time_ms; END; -$$; \ No newline at end of file +$$; From ebb29ca2007088776857c17cadf98a116938513a Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 7 Nov 2022 18:17:12 +0200 Subject: [PATCH 12/50] Refactoring code and optimize import --- .../server/service/ttl/EdgeEventsCleanUpService.java | 1 - .../src/test/resources/application-test.properties | 2 +- .../server/dao/edge/BaseEdgeEventService.java | 7 ++----- .../server/dao/sql/edge/JpaBaseEdgeEventDao.java | 11 +++++------ 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java index 880d3eb125..375c9b3932 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.thingsboard.server.dao.edge.EdgeEventDao; import org.thingsboard.server.dao.edge.EdgeEventService; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import org.thingsboard.server.queue.discovery.PartitionService; diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index fb2fdfe0a4..e4a36c151d 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -62,4 +62,4 @@ sql.audit_logs.partition_size=24 sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 -sql.ttl.edge_events.edge_event_ttl=2592000 \ No newline at end of file +sql.ttl.edge_events.edge_event_ttl=2592000 diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java index 09322aa555..be78866e7a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/BaseEdgeEventService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.edge; import com.google.common.util.concurrent.ListenableFuture; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.edge.EdgeEvent; @@ -27,17 +28,13 @@ import org.thingsboard.server.dao.service.DataValidator; @Service @Slf4j +@AllArgsConstructor public class BaseEdgeEventService implements EdgeEventService { private final EdgeEventDao edgeEventDao; private final DataValidator edgeEventValidator; - public BaseEdgeEventService(EdgeEventDao edgeEventDao, DataValidator edgeEventValidator) { - this.edgeEventDao = edgeEventDao; - this.edgeEventValidator = edgeEventValidator; - } - @Override public ListenableFuture saveAsync(EdgeEvent edgeEvent) { edgeEventValidator.validate(edgeEvent, EdgeEvent::getTenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index a0efcde00e..6f194d03cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -50,6 +50,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; + @Component @SqlDao @RequiredArgsConstructor @@ -223,12 +224,10 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao Date: Mon, 7 Nov 2022 18:24:05 +0200 Subject: [PATCH 13/50] Fix header --- .../server/dao/sql/edge/JpaBaseEdgeEventDao.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index 6f194d03cb..ada3870f97 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 22118af4db7cc2fba246dc8f82f313983a46121a Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 8 Nov 2022 13:13:58 +0200 Subject: [PATCH 14/50] Added Edge RPC response/request support --- .../device/DeviceActorMessageProcessor.java | 5 +- .../service/edge/rpc/EdgeGrpcSession.java | 5 +- .../rpc/constructor/DeviceMsgConstructor.java | 46 +++++-- .../rpc/processor/DeviceEdgeProcessor.java | 113 ++++++++++++++++-- .../rpc/processor/EdgeRpcRequestMetadata.java | 28 +++++ .../server/edge/BaseDeviceEdgeTest.java | 3 +- .../server/common/data/DataConstants.java | 2 + .../common/data/edge/EdgeEventActionType.java | 3 +- common/edge-api/src/main/proto/edge.proto | 3 + .../rule/engine/api/RuleEngineRpcService.java | 2 - .../rule/engine/rpc/TbSendRPCReplyNode.java | 62 +++++++++- 11 files changed, 244 insertions(+), 28 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 2ffe2002f9..9b95b188d2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -823,8 +823,11 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { body.put("expirationTime", msg.getExpirationTime()); body.put("method", msg.getBody().getMethod()); body.put("params", msg.getBody().getParams()); + body.put("persisted", msg.isPersisted()); + body.put("retries", msg.getRetries()); + body.put("additionalInfo", msg.getAdditionalInfo()); - EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(tenantId, edgeId, EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL, deviceId, body); + EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(tenantId, edgeId, EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL_REQUEST, deviceId, body); return Futures.transform(systemContext.getEdgeEventService().saveAsync(edgeEvent), unused -> { systemContext.getClusterService().onEdgeEventUpdate(tenantId, edgeId); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 2342997c65..aca2f8b562 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -463,7 +463,8 @@ public final class EdgeGrpcSession implements Closeable { case UNASSIGNED_FROM_CUSTOMER: case CREDENTIALS_REQUEST: case ENTITY_MERGE_REQUEST: - case RPC_CALL: + case RPC_CALL_REQUEST: + case RPC_CALL_RESPONSE: downlinkMsg = convertEntityEventToDownlink(edgeEvent); log.trace("[{}][{}] entity message processed [{}]", edgeEvent.getTenantId(), this.sessionId, downlinkMsg); break; @@ -611,7 +612,7 @@ public final class EdgeGrpcSession implements Closeable { } if (uplinkMsg.getDeviceRpcCallMsgCount() > 0) { for (DeviceRpcCallMsg deviceRpcCallMsg : uplinkMsg.getDeviceRpcCallMsgList()) { - result.add(ctx.getDeviceProcessor().processDeviceRpcCallResponseFromEdge(edge.getTenantId(), deviceRpcCallMsg)); + result.add(ctx.getDeviceProcessor().processDeviceRpcCallFromEdge(edge.getTenantId(), edge, deviceRpcCallMsg)); } } if (uplinkMsg.getWidgetBundleTypesRequestMsgCount() > 0) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java index 511910dd8a..49438792fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; import org.thingsboard.server.gen.edge.v1.RpcRequestMsg; +import org.thingsboard.server.gen.edge.v1.RpcResponseMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -96,26 +97,51 @@ public class DeviceMsgConstructor { .setIdLSB(deviceId.getId().getLeastSignificantBits()).build(); } - public DeviceRpcCallMsg constructDeviceRpcCallMsg(UUID deviceId, JsonNode body) { - int requestId = body.get("requestId").asInt(); - boolean oneway = body.get("oneway").asBoolean(); - UUID requestUUID = UUID.fromString(body.get("requestUUID").asText()); - long expirationTime = body.get("expirationTime").asLong(); + public DeviceRpcCallMsg constructDeviceRpcRequestMsg(UUID deviceId, JsonNode body) { + DeviceRpcCallMsg.Builder builder = constructDeviceRpcMsg(deviceId, body); + String method = body.get("method").asText(); String params = body.get("params").asText(); - RpcRequestMsg.Builder requestBuilder = RpcRequestMsg.newBuilder(); requestBuilder.setMethod(method); requestBuilder.setParams(params); - DeviceRpcCallMsg.Builder builder = DeviceRpcCallMsg.newBuilder() + builder.setRequestMsg(requestBuilder.build()); + + return builder.build(); + } + + public DeviceRpcCallMsg constructDeviceRpcResponseMsg(UUID deviceId, JsonNode body) { + DeviceRpcCallMsg.Builder builder = constructDeviceRpcMsg(deviceId, body); + + RpcResponseMsg.Builder responseBuilder = RpcResponseMsg.newBuilder(); + if (body.has("error")) { + responseBuilder.setError(body.get("error").asText()); + } else { + responseBuilder.setResponse(body.get("response").asText()); + } + builder.setResponseMsg(responseBuilder.build()); + + return builder.build(); + } + + private DeviceRpcCallMsg.Builder constructDeviceRpcMsg(UUID deviceId, JsonNode body) { + int requestId = body.get("requestId").asInt(); + boolean oneway = body.get("oneway").asBoolean(); + UUID requestUUID = UUID.fromString(body.get("requestUUID").asText()); + long expirationTime = body.get("expirationTime").asLong(); + boolean persisted = body.get("persisted").asBoolean(); + int retries = body.get("retries").asInt(); + String additionalInfo = body.get("additionalInfo").asText(); + return DeviceRpcCallMsg.newBuilder() .setDeviceIdMSB(deviceId.getMostSignificantBits()) .setDeviceIdLSB(deviceId.getLeastSignificantBits()) .setRequestUuidMSB(requestUUID.getMostSignificantBits()) .setRequestUuidLSB(requestUUID.getLeastSignificantBits()) - .setRequestId(requestId) .setExpirationTime(expirationTime) + .setRequestId(requestId) .setOneway(oneway) - .setRequestMsg(requestBuilder.build()); - return builder.build(); + .setPersisted(persisted) + .setRetries(retries) + .setAdditionalInfo(additionalInfo); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java index e09e036953..235fef7431 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -52,6 +53,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; @@ -66,8 +68,14 @@ import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.rpc.FromDeviceRpcResponseActorMsg; +import javax.annotation.PostConstruct; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @Component @@ -75,11 +83,19 @@ import java.util.concurrent.locks.ReentrantLock; @TbCoreComponent public class DeviceEdgeProcessor extends BaseEdgeProcessor { + private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); + private ScheduledExecutorService scheduler; + @Autowired private DataDecodingEncodingService dataDecodingEncodingService; private static final ReentrantLock deviceCreationLock = new ReentrantLock(); + @PostConstruct + public void init(){ + this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-edge-processor-scheduler")); + } + public ListenableFuture processDeviceFromEdge(TenantId tenantId, Edge edge, DeviceUpdateMsg deviceUpdateMsg) { log.trace("[{}] onDeviceUpdate [{}] from edge [{}]", tenantId, deviceUpdateMsg, edge.getName()); switch (deviceUpdateMsg.getMsgType()) { @@ -325,8 +341,17 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { return metaData; } - public ListenableFuture processDeviceRpcCallResponseFromEdge(TenantId tenantId, DeviceRpcCallMsg deviceRpcCallMsg) { - log.trace("[{}] processDeviceRpcCallResponseMsg [{}]", tenantId, deviceRpcCallMsg); + public ListenableFuture processDeviceRpcCallFromEdge(TenantId tenantId, Edge edge, DeviceRpcCallMsg deviceRpcCallMsg) { + log.trace("[{}] processDeviceRpcCallFromEdge [{}]", tenantId, deviceRpcCallMsg); + if (deviceRpcCallMsg.hasResponseMsg()) { + return processDeviceRpcResponseFromEdge(tenantId, deviceRpcCallMsg); + } else if (deviceRpcCallMsg.hasRequestMsg()) { + return processDeviceRpcRequestFromEdge(tenantId, edge, deviceRpcCallMsg); + } + return Futures.immediateFuture(null); + } + + private ListenableFuture processDeviceRpcResponseFromEdge(TenantId tenantId, DeviceRpcCallMsg deviceRpcCallMsg) { SettableFuture futureToSet = SettableFuture.create(); UUID requestUuid = new UUID(deviceRpcCallMsg.getRequestUuidMSB(), deviceRpcCallMsg.getRequestUuidLSB()); DeviceId deviceId = new DeviceId(new UUID(deviceRpcCallMsg.getDeviceIdMSB(), deviceRpcCallMsg.getDeviceIdLSB())); @@ -357,6 +382,68 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { return futureToSet; } + private ListenableFuture processDeviceRpcRequestFromEdge(TenantId tenantId, Edge edge, DeviceRpcCallMsg deviceRpcCallMsg) { + DeviceId deviceId = new DeviceId(new UUID(deviceRpcCallMsg.getDeviceIdMSB(), deviceRpcCallMsg.getDeviceIdLSB())); + UUID requestUUID = new UUID(deviceRpcCallMsg.getRequestUuidMSB(), deviceRpcCallMsg.getRequestUuidLSB()); + try { + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.createObjectNode(); + TbMsgMetaData metaData = new TbMsgMetaData(); + String requestId = Integer.toString(deviceRpcCallMsg.getRequestId()); + metaData.putValue("requestId", requestId); + metaData.putValue("requestUUID", requestUUID.toString()); + // ?? metaData.putValue("originServiceId", deviceRpcRequestMsg.get); + metaData.putValue("expirationTime", Long.toString(deviceRpcCallMsg.getExpirationTime())); + metaData.putValue("oneway", Boolean.toString(deviceRpcCallMsg.getOneway())); + metaData.putValue(DataConstants.PERSISTENT, Boolean.toString(deviceRpcCallMsg.getPersisted())); + + if (deviceRpcCallMsg.getRetries() > 0) { + metaData.putValue(DataConstants.RETRIES, Integer.toString(deviceRpcCallMsg.getRetries())); + } + + metaData.putValue(DataConstants.EDGE_ID, edge.getId().toString()); + + Device device = deviceService.findDeviceById(tenantId, deviceId); + if (device != null) { + metaData.putValue("deviceName", device.getName()); + metaData.putValue("deviceType", device.getType()); + metaData.putValue(DataConstants.DEVICE_ID, deviceId.getId().toString()); + } + + entityNode.put("method", deviceRpcCallMsg.getRequestMsg().getMethod()); + entityNode.put("params", deviceRpcCallMsg.getRequestMsg().getParams()); + + entityNode.put(DataConstants.ADDITIONAL_INFO, deviceRpcCallMsg.getAdditionalInfo()); + TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, null, metaData, + TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("Successfully send ENTITY_CREATED EVENT to rule engine [{}]", device); + } + + @Override + public void onFailure(Throwable t) { + log.debug("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", device, t); + } + }); + toServerRpcPendingMap.put(requestId, new EdgeRpcRequestMetadata(tenantId, edge.getId(), deviceId)); + scheduler.schedule(() -> processTimeout(requestId), 60000, TimeUnit.MILLISECONDS); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("[{}] Failed to push device action to rule engine: {}", deviceId, DataConstants.ENTITY_CREATED, e); + } + + return Futures.immediateFuture(null); + } + + private void processTimeout(String requestId) { + EdgeRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); + if (data != null) { + // TODO: add failure body + saveEdgeEvent(data.getTenantId(), data.getEdgeId(), EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL_RESPONSE, + data.getDeviceId(), JacksonUtil.OBJECT_MAPPER.valueToTree("{}")); + } + } + public DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent) { DeviceId deviceId = new DeviceId(edgeEvent.getEntityId()); DownlinkMsg downlinkMsg = null; @@ -401,8 +488,10 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { .build(); } break; - case RPC_CALL: - return convertRpcCallEventToDownlink(edgeEvent); + case RPC_CALL_REQUEST: + return convertRpcCallRequestEventToDownlink(edgeEvent); + case RPC_CALL_RESPONSE: + return convertRpcCallResponseEventToDownlink(edgeEvent); case CREDENTIALS_REQUEST: return convertCredentialsRequestEventToDownlink(edgeEvent); case ENTITY_MERGE_REQUEST: @@ -411,10 +500,18 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { return downlinkMsg; } - private DownlinkMsg convertRpcCallEventToDownlink(EdgeEvent edgeEvent) { - log.trace("Executing convertRpcCallEventToDownlink, edgeEvent [{}]", edgeEvent); - DeviceRpcCallMsg deviceRpcCallMsg = - deviceMsgConstructor.constructDeviceRpcCallMsg(edgeEvent.getEntityId(), edgeEvent.getBody()); + private DownlinkMsg convertRpcCallRequestEventToDownlink(EdgeEvent edgeEvent) { + log.trace("Executing convertRpcCallRequestEventToDownlink, edgeEvent [{}]", edgeEvent); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addDeviceRpcCallMsg(deviceMsgConstructor.constructDeviceRpcRequestMsg(edgeEvent.getEntityId(), edgeEvent.getBody())) + .build(); + } + + private DownlinkMsg convertRpcCallResponseEventToDownlink(EdgeEvent edgeEvent) { + log.trace("Executing convertRpcCallResponseEventToDownlink, edgeEvent [{}]", edgeEvent); + DeviceRpcCallMsg deviceRpcCallMsg = deviceMsgConstructor.constructDeviceRpcResponseMsg(edgeEvent.getEntityId(), edgeEvent.getBody()); + toServerRpcPendingMap.remove(Integer.toString(deviceRpcCallMsg.getRequestId())); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceRpcCallMsg(deviceRpcCallMsg) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java new file mode 100644 index 0000000000..a5a71d0743 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor; + +import lombok.Data; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class EdgeRpcRequestMetadata { + private final TenantId tenantId; + private final EdgeId edgeId; + private final DeviceId deviceId; +} diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java index 392a6814d4..8e8c616d7a 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java @@ -512,7 +512,8 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest { body.put("method", "test_method"); body.put("params", "{\"param1\":\"value1\"}"); - EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.RPC_CALL, device.getId().getId(), EdgeEventType.DEVICE, body); + EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.RPC_CALL_REQUEST, + device.getId().getId(), EdgeEventType.DEVICE, body); edgeImitator.expectMessageAmount(1); edgeEventService.saveAsync(edgeEvent).get(); clusterService.onEdgeEventUpdate(tenantId, edge.getId()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 85b9e681a1..69c36697d3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -40,6 +40,8 @@ public class DataConstants { public static final String EXPIRATION_TIME = "expirationTime"; public static final String ADDITIONAL_INFO = "additionalInfo"; public static final String RETRIES = "retries"; + public static final String EDGE_ID = "edgeId"; + public static final String DEVICE_ID = "deviceId"; public static final String COAP_TRANSPORT_NAME = "COAP"; public static final String LWM2M_TRANSPORT_NAME = "LWM2M"; public static final String MQTT_TRANSPORT_NAME = "MQTT"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java index bba8767fca..7fae6c56b1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java @@ -28,7 +28,8 @@ public enum EdgeEventActionType { UNASSIGNED_FROM_CUSTOMER, RELATION_ADD_OR_UPDATE, RELATION_DELETED, - RPC_CALL, + RPC_CALL_REQUEST, + RPC_CALL_RESPONSE, ALARM_ACK, ALARM_CLEAR, ASSIGNED_TO_EDGE, diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index a2e3b5989b..a5785d6870 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -430,6 +430,9 @@ message DeviceRpcCallMsg { bool oneway = 7; RpcRequestMsg requestMsg = 8; RpcResponseMsg responseMsg = 9; + bool persisted = 10; + int32 retries = 11; + string additionalInfo = 12; } message RpcRequestMsg { diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java index abb5ace0f8..85596d7e7c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineRpcService.java @@ -15,8 +15,6 @@ */ package org.thingsboard.rule.engine.api; -import org.thingsboard.server.common.data.id.DeviceId; - import java.util.UUID; import java.util.function.Consumer; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index ce9b5050f0..22d13f196b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -15,15 +15,27 @@ */ package org.thingsboard.rule.engine.rpc; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.data.StringUtils; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; @@ -65,9 +77,53 @@ public class TbSendRPCReplyNode implements TbNode { } else if (StringUtils.isEmpty(msg.getData())) { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { - ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); - ctx.tellSuccess(msg); + if (StringUtils.isNotBlank(msg.getMetaData().getValue(DataConstants.EDGE_ID))) { + saveRpcResponseToEdgeQueue(ctx, msg); + } else { + ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); + ctx.tellSuccess(msg); + } } } + private void saveRpcResponseToEdgeQueue(TbContext ctx, TbMsg msg) { +// EdgeEvent edgeEvent = new EdgeEvent(); +// edgeEvent.setTenantId(tenantId); +// edgeEvent.setAction(eventAction); +// edgeEvent.setEntityId(entityId); +// edgeEvent.setType(eventType); +// edgeEvent.setBody(entityBody); +// edgeEvent.setEdgeId(edgeId); +// +// ObjectNode body = mapper.createObjectNode(); +// body.put("requestId", requestId); +// body.put("requestUUID", msg.getId().toString()); +// body.put("oneway", msg.isOneway()); +// body.put("expirationTime", msg.getExpirationTime()); +// body.put("method", msg.getBody().getMethod()); +// body.put("params", msg.getBody().getParams()); +// body.put("persisted", msg.isPersisted()); +// body.put("retries", msg.getRetries()); +// body.put("additionalInfo", msg.getAdditionalInfo()); + + EdgeId edgeId = new EdgeId(UUID.fromString(msg.getMetaData().getValue(DataConstants.EDGE_ID))); + DeviceId deviceId = new DeviceId(UUID.fromString(msg.getMetaData().getValue(DataConstants.DEVICE_ID))); + // TODO: add body + EdgeEvent edgeEvent = + EdgeUtils.constructEdgeEvent(ctx.getTenantId(), edgeId, EdgeEventType.DEVICE, + EdgeEventActionType.RPC_CALL_RESPONSE, deviceId, JacksonUtil.OBJECT_MAPPER.valueToTree("{}")); + + ListenableFuture future = ctx.getEdgeEventService().saveAsync(edgeEvent); + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + ctx.onEdgeEventUpdate(ctx.getTenantId(), edgeId); + ctx.tellSuccess(msg); + } + + @Override + public void onFailure(Throwable t) { + } + }, ctx.getDbCallbackExecutor()); + } } From 2d2d78ae6bc656d9ed36375743cfcf5455e9e7b7 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 8 Nov 2022 15:08:22 +0100 Subject: [PATCH 15/50] fetch attribute types for upgrade improvements --- .../install/SqlDatabaseUpgradeService.java | 39 +++++++++++-------- .../server/dao/asset/AssetDao.java | 3 ++ .../server/dao/model/sql/TbPair.java | 26 +++++++++++++ .../server/dao/sql/asset/AssetRepository.java | 18 +++++---- .../server/dao/sql/asset/JpaAssetDao.java | 9 +++++ 5 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 4b6f1e1a52..3067e545b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -34,12 +34,13 @@ import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.model.sql.TbPair; import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.sql.asset.AssetRepository; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -57,7 +58,10 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO; @@ -111,7 +115,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService private DeviceService deviceService; @Autowired - private AssetRepository assetRepository; + private AssetDao assetDao; @Autowired private DeviceProfileService deviceProfileService; @@ -619,24 +623,27 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService loadSql(schemaUpdateFile, conn); log.info("Creating default asset profiles..."); - PageLink pageLink = new PageLink(100); - PageData pageData; + PageLink pageLink = new PageLink(1000); + PageData> pageData; List> futures = new ArrayList<>(); + Set tenants = new HashSet<>(); do { - pageData = tenantService.findTenants(pageLink); - for (Tenant tenant : pageData.getData()) { - List assetTypes = assetRepository.findTenantAssetTypes(tenant.getUuidId()); - assetTypes.remove("default"); - - try { - futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.createDefaultAssetProfile(tenant.getId()))); - } catch (Exception e) { + pageData = assetDao.getAllAssetTypes(pageLink); + for (TbPair pair : pageData.getData()) { + TenantId tenantId = new TenantId(pair.getFirst()); + String assetType = pair.getSecond(); + if (tenants.add(tenantId)) { + try { + futures.add(dbUpgradeExecutor.submit(() -> + assetProfileService.createDefaultAssetProfile(tenantId))); + } catch (Exception e) {} } - for (String assetType : assetTypes) { + + if (!"default".equals(assetType)) { try { - futures.add(dbUpgradeExecutor.submit(() -> assetProfileService.findOrCreateAssetProfile(tenant.getId(), assetType))); - } catch (Exception e) { - } + futures.add(dbUpgradeExecutor.submit(() -> + assetProfileService.findOrCreateAssetProfile(tenantId, assetType))); + } catch (Exception e) {} } } pageLink = pageLink.nextPageLink(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 4d67a794b6..d17181cffa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.model.sql.TbPair; import java.util.List; import java.util.Optional; @@ -222,4 +223,6 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD * @return the list of asset objects */ PageData findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink); + + PageData> getAllAssetTypes(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java new file mode 100644 index 0000000000..4cea485575 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class TbPair { + private S first; + private T second; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index eadf0d2b84..cbb68b6ccc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -23,6 +23,7 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; +import org.thingsboard.server.dao.model.sql.TbPair; import java.util.List; import java.util.UUID; @@ -70,9 +71,9 @@ public interface AssetRepository extends JpaRepository, Expor "AND a.assetProfileId = :profileId " + "AND LOWER(a.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") Page findByTenantIdAndProfileId(@Param("tenantId") UUID tenantId, - @Param("profileId") UUID profileId, - @Param("searchText") String searchText, - Pageable pageable); + @Param("profileId") UUID profileId, + @Param("searchText") String searchText, + Pageable pageable); @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo, p.name) " + "FROM AssetEntity a " + @@ -186,14 +187,17 @@ public interface AssetRepository extends JpaRepository, Expor "AND a.type = :type " + "AND LOWER(a.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") Page findByTenantIdAndEdgeIdAndType(@Param("tenantId") UUID tenantId, - @Param("edgeId") UUID edgeId, - @Param("type") String type, - @Param("searchText") String searchText, - Pageable pageable); + @Param("edgeId") UUID edgeId, + @Param("type") String type, + @Param("searchText") String searchText, + Pageable pageable); Long countByTenantIdAndTypeIsNot(UUID tenantId, String type); @Query("SELECT externalId FROM AssetEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); + @Query(value = "SELECT DISTINCT new org.thingsboard.server.dao.model.sql.TbPair(a.tenantId , a.type) FROM AssetEntity a") + Page> getAllAssetTypes(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 489e15502e..a0c70428a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -28,14 +28,17 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; +import org.thingsboard.server.dao.model.sql.TbPair; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -243,6 +246,12 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im DaoUtil.toPageable(pageLink))); } + public PageData> getAllAssetTypes(PageLink pageLink) { + log.debug("Try to find all asset types and pageLink [{}]", pageLink); + return DaoUtil.pageToPageData(assetRepository.getAllAssetTypes( + DaoUtil.toPageable(pageLink, Arrays.asList(new SortOrder("tenantId"), new SortOrder("type"))))); + } + @Override public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE); From ff3f0234a94f979053ade94425e5fb733ba5ee46 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Tue, 8 Nov 2022 17:46:11 +0200 Subject: [PATCH 16/50] Refactoring the code according to the comments after the review --- .../install/update/DefaultDataUpdateService.java | 15 ++++++--------- .../service/ttl/EdgeEventsCleanUpService.java | 2 +- .../server/dao/sql/edge/JpaBaseEdgeEventDao.java | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index ea6d278844..be8f1db3df 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -49,11 +49,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.queue.ProcessingStrategy; -import org.thingsboard.server.common.data.queue.ProcessingStrategyType; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.data.queue.SubmitStrategy; -import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.data.queue.*; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; @@ -186,16 +182,17 @@ public class DefaultDataUpdateService implements DataUpdateService { break; case "3.4.1": boolean skipAuditLogsMigration = getEnv("TB_SKIP_AUDIT_LOGS_MIGRATION", false); - if (!skipAuditLogsMigration) { + boolean skipEdgeEventsMigration = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); + if (!skipAuditLogsMigration || !skipEdgeEventsMigration) { log.info("Updating data from version 3.4.1 to 3.4.2 ..."); + } + if (!skipAuditLogsMigration) { log.info("Starting audit logs migration. Can be skipped with TB_SKIP_AUDIT_LOGS_MIGRATION env variable set to true"); auditLogDao.migrateAuditLogs(); } else { log.info("Skipping audit logs migration"); } - boolean skipEdgeEventsMigrationTemp = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); - if (!skipEdgeEventsMigrationTemp) { - log.info("Updating data from version 3.4.1 to 3.4.2 ..."); + if (!skipEdgeEventsMigration) { log.info("Starting edge events migration. Can be skipped with TB_SKIP_EDGE_EVENTS_MIGRATION env variable set to true"); edgeEventDao.migrateEdgeEvents(); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java index 375c9b3932..a56b1963d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EdgeEventsCleanUpService.java @@ -60,7 +60,7 @@ public class EdgeEventsCleanUpService extends AbstractCleanUpService { @Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.edge_events.execution_interval_ms}") public void cleanUp() { long edgeEventsExpTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttl); - if(isSystemTenantPartitionMine()) { + if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { edgeEventService.cleanupEvents(edgeEventsExpTime); } else { partitioningRepository.cleanupPartitionsCache(EDGE_EVENT_COLUMN_FAMILY_NAME, edgeEventsExpTime, TimeUnit.HOURS.toMillis(partitionSizeInHours)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index ada3870f97..3ab21ae6ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -203,7 +203,7 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao 0 ? System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(edge_events_ttl) : 1480982400000L; + long startTime = edge_events_ttl > 0 ? System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(edge_events_ttl) : 1629158400000L; long currentTime = System.currentTimeMillis(); var partitionStepInMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); From 0849aa722f8f3b310d57a1be3db233f8dee479fc Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 8 Nov 2022 17:34:44 +0100 Subject: [PATCH 17/50] delete assets with a non-existent tenant and set default asset profile to the assets with xss --- .../server/service/install/SqlDatabaseUpgradeService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 3067e545b1..777284521e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -622,6 +622,8 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.1", "schema_update_before.sql"); loadSql(schemaUpdateFile, conn); + conn.createStatement().execute("DELETE FROM asset a WHERE NOT exists(SELECT id FROM tenant WHERE id = a.tenant_id);"); + log.info("Creating default asset profiles..."); PageLink pageLink = new PageLink(1000); PageData> pageData; @@ -654,6 +656,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.info("Updating asset profiles..."); conn.createStatement().execute("call update_asset_profiles()"); + conn.createStatement().execute("UPDATE asset a SET asset_profile_id = " + + "(SELECT id FROM asset_profile ap WHERE ap.tenant_id = a.tenant_id AND name='default') WHERE a.asset_profile_id IS NULL;"); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.1", "schema_update_after.sql"); loadSql(schemaUpdateFile, conn); From 71d26ba18b83edd9b17a5f5b9cf0a8a9df78886d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 9 Nov 2022 01:18:01 +0100 Subject: [PATCH 18/50] asset profile upgrade improvements --- .../install/SqlDatabaseUpgradeService.java | 36 +++++++++++++------ .../dao/sql/tenant/TenantRepository.java | 4 +++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 777284521e..bcd3686183 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.model.sql.TbPair; import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.sql.tenant.TenantRepository; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; @@ -111,6 +112,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @Autowired private TenantService tenantService; + @Autowired + private TenantRepository tenantRepository; + @Autowired private DeviceService deviceService; @@ -628,29 +632,41 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService PageLink pageLink = new PageLink(1000); PageData> pageData; List> futures = new ArrayList<>(); - Set tenants = new HashSet<>(); + Set tenants = new HashSet<>(); do { pageData = assetDao.getAllAssetTypes(pageLink); for (TbPair pair : pageData.getData()) { TenantId tenantId = new TenantId(pair.getFirst()); String assetType = pair.getSecond(); - if (tenants.add(tenantId)) { - try { - futures.add(dbUpgradeExecutor.submit(() -> - assetProfileService.createDefaultAssetProfile(tenantId))); - } catch (Exception e) {} + if (tenants.add(pair.getFirst())) { + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.createDefaultAssetProfile(tenantId); + } catch (Exception e) {} + })); } if (!"default".equals(assetType)) { - try { - futures.add(dbUpgradeExecutor.submit(() -> - assetProfileService.findOrCreateAssetProfile(tenantId, assetType))); - } catch (Exception e) {} + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.findOrCreateAssetProfile(tenantId, assetType); + } catch (Exception e) {} + })); } } pageLink = pageLink.nextPageLink(); } while (pageData.hasNext()); + List tenantsWithoutProfiles = tenantRepository.getIdsNotIn(tenants); + + tenantsWithoutProfiles.forEach(uuid -> + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.createDefaultAssetProfile(TenantId.fromUUID(uuid)); + } catch (Exception e) {} + })) + ); + Futures.allAsList(futures).get(); log.info("Updating asset profiles..."); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 7042dcbb16..fb563895d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -24,6 +24,7 @@ import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; import java.util.List; +import java.util.Set; import java.util.UUID; /** @@ -54,4 +55,7 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); + @Query("SELECT t.id FROM TenantEntity t WHERE t.id NOT IN :ids") + List getIdsNotIn(@Param("ids")Set ids); + } From 545790fc5b8430357f1f5a47291a92eda36b5f5d Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 9 Nov 2022 10:18:20 +0200 Subject: [PATCH 19/50] Do not register timeout for edge processor on cloud --- .../rpc/processor/DeviceEdgeProcessor.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java index 235fef7431..b340b252ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java @@ -25,7 +25,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -68,14 +67,8 @@ import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.rpc.FromDeviceRpcResponseActorMsg; -import javax.annotation.PostConstruct; -import java.util.Map; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @Component @@ -83,19 +76,11 @@ import java.util.concurrent.locks.ReentrantLock; @TbCoreComponent public class DeviceEdgeProcessor extends BaseEdgeProcessor { - private final Map toServerRpcPendingMap = new ConcurrentHashMap<>(); - private ScheduledExecutorService scheduler; - @Autowired private DataDecodingEncodingService dataDecodingEncodingService; private static final ReentrantLock deviceCreationLock = new ReentrantLock(); - @PostConstruct - public void init(){ - this.scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("device-edge-processor-scheduler")); - } - public ListenableFuture processDeviceFromEdge(TenantId tenantId, Edge edge, DeviceUpdateMsg deviceUpdateMsg) { log.trace("[{}] onDeviceUpdate [{}] from edge [{}]", tenantId, deviceUpdateMsg, edge.getName()); switch (deviceUpdateMsg.getMsgType()) { @@ -426,8 +411,6 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { log.debug("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", device, t); } }); - toServerRpcPendingMap.put(requestId, new EdgeRpcRequestMetadata(tenantId, edge.getId(), deviceId)); - scheduler.schedule(() -> processTimeout(requestId), 60000, TimeUnit.MILLISECONDS); } catch (JsonProcessingException | IllegalArgumentException e) { log.warn("[{}] Failed to push device action to rule engine: {}", deviceId, DataConstants.ENTITY_CREATED, e); } @@ -435,15 +418,6 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { return Futures.immediateFuture(null); } - private void processTimeout(String requestId) { - EdgeRpcRequestMetadata data = toServerRpcPendingMap.remove(requestId); - if (data != null) { - // TODO: add failure body - saveEdgeEvent(data.getTenantId(), data.getEdgeId(), EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL_RESPONSE, - data.getDeviceId(), JacksonUtil.OBJECT_MAPPER.valueToTree("{}")); - } - } - public DownlinkMsg convertDeviceEventToDownlink(EdgeEvent edgeEvent) { DeviceId deviceId = new DeviceId(edgeEvent.getEntityId()); DownlinkMsg downlinkMsg = null; @@ -511,7 +485,6 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { private DownlinkMsg convertRpcCallResponseEventToDownlink(EdgeEvent edgeEvent) { log.trace("Executing convertRpcCallResponseEventToDownlink, edgeEvent [{}]", edgeEvent); DeviceRpcCallMsg deviceRpcCallMsg = deviceMsgConstructor.constructDeviceRpcResponseMsg(edgeEvent.getEntityId(), edgeEvent.getBody()); - toServerRpcPendingMap.remove(Integer.toString(deviceRpcCallMsg.getRequestId())); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addDeviceRpcCallMsg(deviceRpcCallMsg) From d84fd92f571b93b3d789421e7a0fb3aa2aec75b4 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 9 Nov 2022 09:58:15 +0100 Subject: [PATCH 20/50] refactored --- .../install/SqlDatabaseUpgradeService.java | 43 +++++++++---------- .../dao/sql/tenant/TenantRepository.java | 5 --- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index bcd3686183..d96475ad31 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -629,23 +629,30 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService conn.createStatement().execute("DELETE FROM asset a WHERE NOT exists(SELECT id FROM tenant WHERE id = a.tenant_id);"); log.info("Creating default asset profiles..."); - PageLink pageLink = new PageLink(1000); - PageData> pageData; List> futures = new ArrayList<>(); + + PageLink pageLink = new PageLink(1000); + PageData tenantIds; + do { + tenantIds = tenantService.findTenantsIds(pageLink); + for (TenantId tenantId : tenantIds.getData()) { + futures.add(dbUpgradeExecutor.submit(() -> { + try { + assetProfileService.createDefaultAssetProfile(tenantId); + } catch (Exception e) {} + })); + } + pageLink = pageLink.nextPageLink(); + } while (tenantIds.hasNext()); + + pageLink = new PageLink(1000); + PageData> pairs; Set tenants = new HashSet<>(); do { - pageData = assetDao.getAllAssetTypes(pageLink); - for (TbPair pair : pageData.getData()) { + pairs = assetDao.getAllAssetTypes(pageLink); + for (TbPair pair : pairs.getData()) { TenantId tenantId = new TenantId(pair.getFirst()); String assetType = pair.getSecond(); - if (tenants.add(pair.getFirst())) { - futures.add(dbUpgradeExecutor.submit(() -> { - try { - assetProfileService.createDefaultAssetProfile(tenantId); - } catch (Exception e) {} - })); - } - if (!"default".equals(assetType)) { futures.add(dbUpgradeExecutor.submit(() -> { try { @@ -655,17 +662,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } pageLink = pageLink.nextPageLink(); - } while (pageData.hasNext()); - - List tenantsWithoutProfiles = tenantRepository.getIdsNotIn(tenants); - - tenantsWithoutProfiles.forEach(uuid -> - futures.add(dbUpgradeExecutor.submit(() -> { - try { - assetProfileService.createDefaultAssetProfile(TenantId.fromUUID(uuid)); - } catch (Exception e) {} - })) - ); + } while (pairs.hasNext()); Futures.allAsList(futures).get(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index fb563895d5..ac02d77bc8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -24,7 +24,6 @@ import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; import java.util.List; -import java.util.Set; import java.util.UUID; /** @@ -54,8 +53,4 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); - - @Query("SELECT t.id FROM TenantEntity t WHERE t.id NOT IN :ids") - List getIdsNotIn(@Param("ids")Set ids); - } From a50fa91728bd0195bce76ec918037c818138f9da Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 9 Nov 2022 11:32:31 +0200 Subject: [PATCH 21/50] Refactoring code --- .../install/update/DefaultDataUpdateService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index be8f1db3df..93f83414ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -49,7 +49,11 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.queue.*; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; @@ -181,17 +185,15 @@ public class DefaultDataUpdateService implements DataUpdateService { } break; case "3.4.1": + log.info("Updating data from version 3.4.1 to 3.4.2 ..."); boolean skipAuditLogsMigration = getEnv("TB_SKIP_AUDIT_LOGS_MIGRATION", false); - boolean skipEdgeEventsMigration = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); - if (!skipAuditLogsMigration || !skipEdgeEventsMigration) { - log.info("Updating data from version 3.4.1 to 3.4.2 ..."); - } if (!skipAuditLogsMigration) { log.info("Starting audit logs migration. Can be skipped with TB_SKIP_AUDIT_LOGS_MIGRATION env variable set to true"); auditLogDao.migrateAuditLogs(); } else { log.info("Skipping audit logs migration"); } + boolean skipEdgeEventsMigration = getEnv("TB_SKIP_EDGE_EVENTS_MIGRATION", false); if (!skipEdgeEventsMigration) { log.info("Starting edge events migration. Can be skipped with TB_SKIP_EDGE_EVENTS_MIGRATION env variable set to true"); edgeEventDao.migrateEdgeEvents(); From 7e8f7a02ca0196ec79e514ff91e136f0c6456d2a Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 9 Nov 2022 12:13:19 +0200 Subject: [PATCH 22/50] Refactoring code --- application/src/main/data/upgrade/3.4.1/schema_update.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/main/data/upgrade/3.4.1/schema_update.sql b/application/src/main/data/upgrade/3.4.1/schema_update.sql index 0f4354c0ca..8dceb0115d 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update.sql @@ -14,6 +14,7 @@ -- limitations under the License. -- +-- AUDIT LOGS MIGRATION START DO $$ DECLARE table_partition RECORD; @@ -73,8 +74,10 @@ BEGIN WHERE created_time >= start_time_ms AND created_time < end_time_ms; END; $$; +-- AUDIT LOGS MIGRATION END +-- EDGE EVENTS MIGRATION START DO $$ DECLARE table_partition RECORD; @@ -131,3 +134,4 @@ BEGIN WHERE created_time >= start_time_ms AND created_time < end_time_ms; END; $$; +-- EDGE EVENTS MIGRATION END From d1312cb917c15eddfc685a8502a67ffcadda3db0 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 9 Nov 2022 12:54:17 +0200 Subject: [PATCH 23/50] Added serviceId and sessionId to deviceRpcCallmsg --- .../rpc/constructor/DeviceMsgConstructor.java | 45 ++++++++++++------- .../rpc/processor/DeviceEdgeProcessor.java | 11 +++-- common/edge-api/src/main/proto/edge.proto | 2 + .../rule/engine/rpc/TbSendRPCReplyNode.java | 20 ++++++--- 4 files changed, 53 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java index 49438792fa..ebea5d482f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java @@ -125,23 +125,36 @@ public class DeviceMsgConstructor { } private DeviceRpcCallMsg.Builder constructDeviceRpcMsg(UUID deviceId, JsonNode body) { - int requestId = body.get("requestId").asInt(); - boolean oneway = body.get("oneway").asBoolean(); - UUID requestUUID = UUID.fromString(body.get("requestUUID").asText()); - long expirationTime = body.get("expirationTime").asLong(); - boolean persisted = body.get("persisted").asBoolean(); - int retries = body.get("retries").asInt(); - String additionalInfo = body.get("additionalInfo").asText(); - return DeviceRpcCallMsg.newBuilder() + DeviceRpcCallMsg.Builder builder = DeviceRpcCallMsg.newBuilder() .setDeviceIdMSB(deviceId.getMostSignificantBits()) .setDeviceIdLSB(deviceId.getLeastSignificantBits()) - .setRequestUuidMSB(requestUUID.getMostSignificantBits()) - .setRequestUuidLSB(requestUUID.getLeastSignificantBits()) - .setExpirationTime(expirationTime) - .setRequestId(requestId) - .setOneway(oneway) - .setPersisted(persisted) - .setRetries(retries) - .setAdditionalInfo(additionalInfo); + .setRequestId(body.get("requestId").asInt()); + if (body.get("oneway") != null) { + builder.setOneway(body.get("oneway").asBoolean()); + } + if (body.get("requestUUID") != null) { + UUID requestUUID = UUID.fromString(body.get("requestUUID").asText()); + builder.setRequestUuidMSB(requestUUID.getMostSignificantBits()) + .setRequestUuidLSB(requestUUID.getLeastSignificantBits()); + } + if (body.get("expirationTime") != null) { + builder.setExpirationTime(body.get("expirationTime").asLong()); + } + if (body.get("persisted") != null) { + builder.setPersisted(body.get("persisted").asBoolean()); + } + if (body.get("retries") != null) { + builder.setRetries(body.get("retries").asInt()); + } + if (body.get("additionalInfo") != null) { + builder.setAdditionalInfo(JacksonUtil.toString(body.get("additionalInfo"))); + } + if (body.get("serviceId") != null) { + builder.setServiceId(body.get("serviceId").asText()); + } + if (body.get("sessionId") != null) { + builder.setSessionId(body.get("sessionId").asText()); + } + return builder; } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java index b340b252ef..959f5c5fe8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java @@ -376,7 +376,8 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { String requestId = Integer.toString(deviceRpcCallMsg.getRequestId()); metaData.putValue("requestId", requestId); metaData.putValue("requestUUID", requestUUID.toString()); - // ?? metaData.putValue("originServiceId", deviceRpcRequestMsg.get); + metaData.putValue("serviceId", deviceRpcCallMsg.getServiceId()); + metaData.putValue("sessionId", deviceRpcCallMsg.getSessionId()); metaData.putValue("expirationTime", Long.toString(deviceRpcCallMsg.getExpirationTime())); metaData.putValue("oneway", Boolean.toString(deviceRpcCallMsg.getOneway())); metaData.putValue(DataConstants.PERSISTENT, Boolean.toString(deviceRpcCallMsg.getPersisted())); @@ -403,16 +404,18 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - log.debug("Successfully send ENTITY_CREATED EVENT to rule engine [{}]", device); + log.debug("Successfully send TO_SERVER_RPC_REQUEST to rule engine [{}], deviceRpcCallMsg {}", + device, deviceRpcCallMsg); } @Override public void onFailure(Throwable t) { - log.debug("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", device, t); + log.debug("Failed to send TO_SERVER_RPC_REQUEST to rule engine [{}], deviceRpcCallMsg {}", + device, deviceRpcCallMsg, t); } }); } catch (JsonProcessingException | IllegalArgumentException e) { - log.warn("[{}] Failed to push device action to rule engine: {}", deviceId, DataConstants.ENTITY_CREATED, e); + log.warn("[{}] Failed to push TO_SERVER_RPC_REQUEST to rule engine. deviceRpcCallMsg {}", deviceId, deviceRpcCallMsg, e); } return Futures.immediateFuture(null); diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index a5785d6870..d70179cfe0 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -433,6 +433,8 @@ message DeviceRpcCallMsg { bool persisted = 10; int32 retries = 11; string additionalInfo = 12; + string serviceId = 13; + string sessionId = 14; } message RpcRequestMsg { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 22d13f196b..89bc55bb54 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -15,6 +15,8 @@ */ package org.thingsboard.rule.engine.rpc; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -78,7 +80,11 @@ public class TbSendRPCReplyNode implements TbNode { ctx.tellFailure(msg, new RuntimeException("Request body is empty!")); } else { if (StringUtils.isNotBlank(msg.getMetaData().getValue(DataConstants.EDGE_ID))) { - saveRpcResponseToEdgeQueue(ctx, msg); + try { + saveRpcResponseToEdgeQueue(ctx, msg, serviceIdStr, sessionIdStr, requestIdStr); + } catch (Exception e) { + ctx.tellFailure(msg, e); + } } else { ctx.getRpcService().sendRpcReplyToDevice(serviceIdStr, UUID.fromString(sessionIdStr), Integer.parseInt(requestIdStr), msg.getData()); ctx.tellSuccess(msg); @@ -86,7 +92,7 @@ public class TbSendRPCReplyNode implements TbNode { } } - private void saveRpcResponseToEdgeQueue(TbContext ctx, TbMsg msg) { + private void saveRpcResponseToEdgeQueue(TbContext ctx, TbMsg msg, String serviceIdStr, String sessionIdStr, String requestIdStr) throws JsonProcessingException { // EdgeEvent edgeEvent = new EdgeEvent(); // edgeEvent.setTenantId(tenantId); // edgeEvent.setAction(eventAction); @@ -95,8 +101,11 @@ public class TbSendRPCReplyNode implements TbNode { // edgeEvent.setBody(entityBody); // edgeEvent.setEdgeId(edgeId); // -// ObjectNode body = mapper.createObjectNode(); -// body.put("requestId", requestId); + ObjectNode body = JacksonUtil.OBJECT_MAPPER.createObjectNode(); + body.put("serviceId", serviceIdStr); + body.put("sessionId", sessionIdStr); + body.put("requestId", requestIdStr); + body.put("response", JacksonUtil.OBJECT_MAPPER.writeValueAsString(msg.getData())); // body.put("requestUUID", msg.getId().toString()); // body.put("oneway", msg.isOneway()); // body.put("expirationTime", msg.getExpirationTime()); @@ -111,7 +120,7 @@ public class TbSendRPCReplyNode implements TbNode { // TODO: add body EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(ctx.getTenantId(), edgeId, EdgeEventType.DEVICE, - EdgeEventActionType.RPC_CALL_RESPONSE, deviceId, JacksonUtil.OBJECT_MAPPER.valueToTree("{}")); + EdgeEventActionType.RPC_CALL_RESPONSE, deviceId, JacksonUtil.OBJECT_MAPPER.valueToTree(body)); ListenableFuture future = ctx.getEdgeEventService().saveAsync(edgeEvent); Futures.addCallback(future, new FutureCallback() { @@ -123,6 +132,7 @@ public class TbSendRPCReplyNode implements TbNode { @Override public void onFailure(Throwable t) { + ctx.tellFailure(msg, t); } }, ctx.getDbCallbackExecutor()); } From 3cb94532fabd40fa67a1394234daad74c48a90f5 Mon Sep 17 00:00:00 2001 From: dlandiak Date: Wed, 9 Nov 2022 14:03:37 +0200 Subject: [PATCH 24/50] get inactivity using entity data query instead of separate attribute fetch request --- .../state/DefaultDeviceStateService.java | 20 ++++------- .../state/DefaultDeviceStateServiceTest.java | 36 +++++++++++-------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 38f4ee50c7..4aaba9c6ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -84,7 +84,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -123,7 +122,8 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService PERSISTENT_ATTRIBUTE_KEYS = Arrays.asList( new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_ACTIVITY_TIME), @@ -662,24 +662,16 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService attributeOpt = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), SERVER_SCOPE, INACTIVITY_TIMEOUT).get(); - attributeOpt.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> { - if (inactivityTimeout > 0) { - deviceStateData.getState().setInactivityTimeout(inactivityTimeout); - } - }); - } catch (Exception e) { - log.warn("[{}] Failed to fetch inactivity timeout from attribute", deviceStateData.getDeviceId(), e); - } + long inactivityTimeout = getEntryValue(ed, EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT, TimeUnit.SECONDS.toMillis(defaultInactivityTimeoutInSec)); + deviceStateData.getState().setInactivityTimeout(inactivityTimeout); } } diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index bc0e067442..b33b7162c4 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.state; -import com.google.common.util.concurrent.Futures; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -27,8 +26,6 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.TsValue; @@ -40,19 +37,14 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import java.util.Map; -import java.util.Optional; import java.util.UUID; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; -import static org.mockito.Mockito.when; -import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_TIMEOUT; @RunWith(MockitoJUnitRunner.class) @@ -104,21 +96,35 @@ public class DefaultDeviceStateServiceTest { @Test public void givenPersistToTelemetryAndDefaultInactivityTimeoutFetched_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() { var defaultInactivityTimeoutInSec = 60L; + var latest = + Map.of( + EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000))), + EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L))) + ); + + process(latest, defaultInactivityTimeoutInSec); + } + + @Test + public void givenPersistToTelemetryAndNoInactivityTimeoutFetchedFromTimeSeries_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() { + var defaultInactivityTimeoutInSec = 60L; + var latest = + Map.of( + EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L))) + ); + + process(latest, defaultInactivityTimeoutInSec); + } + + private void process(Map> latest, long defaultInactivityTimeoutInSec) { service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec); service.setPersistToTelemetry(true); var deviceUuid = UUID.randomUUID(); var deviceId = new DeviceId(deviceUuid); - when(attributesService.find(any(), any(), anyString(), anyString())) - .thenReturn(Futures.immediateFuture(Optional.of(new BaseAttributeKvEntry(0, new LongDataEntry(INACTIVITY_TIMEOUT, 5000L))))); - - var latest = - Map.of(EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000)))); DeviceStateData deviceStateData = service.toDeviceStateData(new EntityData(deviceId, latest, Map.of()), new DeviceIdInfo(TenantId.SYS_TENANT_ID.getId(), UUID.randomUUID(), deviceUuid)); - Mockito.verify(attributesService, times(1)).find(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, INACTIVITY_TIMEOUT); - Assert.assertEquals(5000L, deviceStateData.getState().getInactivityTimeout()); } From 03aafd4c3ca0065d6244ca473cbc9ed63b3b9bad Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 9 Nov 2022 13:28:42 +0100 Subject: [PATCH 25/50] added TbServiceQueue asset validation and upgrade improvements --- .../data/upgrade/3.4.1/schema_update_before.sql | 9 +++++---- .../server/controller/AssetController.java | 5 ----- .../entitiy/asset/DefaultTbAssetService.java | 14 ++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 15 +++++---------- .../server/common/data/util}/TbPair.java | 2 +- .../thingsboard/server/dao/asset/AssetDao.java | 2 +- .../server/dao/sql/asset/AssetRepository.java | 4 ++-- .../server/dao/sql/asset/JpaAssetDao.java | 2 +- 8 files changed, 29 insertions(+), 24 deletions(-) rename {dao/src/main/java/org/thingsboard/server/dao/model/sql => common/data/src/main/java/org/thingsboard/server/common/data/util}/TbPair.java (94%) diff --git a/application/src/main/data/upgrade/3.4.1/schema_update_before.sql b/application/src/main/data/upgrade/3.4.1/schema_update_before.sql index 59566e42b5..27f772aba3 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update_before.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update_before.sql @@ -37,9 +37,10 @@ CREATE OR REPLACE PROCEDURE update_asset_profiles() LANGUAGE plpgsql AS $$ BEGIN -UPDATE asset as a SET asset_profile_id = p.id - FROM - (SELECT id, tenant_id, name from asset_profile) as p -WHERE a.asset_profile_id IS NULL AND p.tenant_id = a.tenant_id AND a.type = p.name; + UPDATE asset a SET asset_profile_id = COALESCE( + (SELECT id from asset_profile p WHERE p.tenant_id = a.tenant_id AND a.type = p.name), + (SELECT id from asset_profile p WHERE p.tenant_id = a.tenant_id AND p.name = 'default') + ) + WHERE a.asset_profile_id IS NULL; END; $$; diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 8450c67ea3..968829b65a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -86,7 +85,6 @@ import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHO import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; import static org.thingsboard.server.controller.EdgeController.EDGE_ID; -import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; @RestController @TbCoreComponent @@ -148,9 +146,6 @@ public class AssetController extends BaseController { @RequestMapping(value = "/asset", method = RequestMethod.POST) @ResponseBody public Asset saveAsset(@ApiParam(value = "A JSON value representing the asset.") @RequestBody Asset asset) throws Exception { - if (TB_SERVICE_QUEUE.equals(asset.getType())) { - throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } asset.setTenantId(getTenantId()); checkEntity(asset.getId(), asset, Resource.ASSET); return tbAssetService.save(asset, getCurrentUser()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index 118dfe3b58..abaef626b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -22,8 +22,10 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; @@ -31,20 +33,32 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; import java.util.List; +import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; + @Service @AllArgsConstructor public class DefaultTbAssetService extends AbstractTbEntityService implements TbAssetService { private final AssetService assetService; + private final TbAssetProfileCache assetProfileCache; @Override public Asset save(Asset asset, User user) throws Exception { ActionType actionType = asset.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = asset.getTenantId(); try { + if (TB_SERVICE_QUEUE.equals(asset.getType())) { + throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } else if (asset.getAssetProfileId() != null) { + AssetProfile assetProfile = assetProfileCache.get(tenantId, asset.getAssetProfileId()); + if (assetProfile != null && TB_SERVICE_QUEUE.equals(assetProfile.getName())) { + throw new ThingsboardException("Unable to save asset with profile " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + } Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); autoCommit(user, savedAsset.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), savedAsset, diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index d96475ad31..80665d544f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -39,7 +39,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.sql.tenant.TenantRepository; import org.thingsboard.server.dao.tenant.TenantService; @@ -59,9 +59,7 @@ import java.sql.SQLSyntaxErrorException; import java.sql.SQLWarning; import java.sql.Statement; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -629,11 +627,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService conn.createStatement().execute("DELETE FROM asset a WHERE NOT exists(SELECT id FROM tenant WHERE id = a.tenant_id);"); log.info("Creating default asset profiles..."); - List> futures = new ArrayList<>(); PageLink pageLink = new PageLink(1000); PageData tenantIds; do { + List> futures = new ArrayList<>(); tenantIds = tenantService.findTenantsIds(pageLink); for (TenantId tenantId : tenantIds.getData()) { futures.add(dbUpgradeExecutor.submit(() -> { @@ -642,13 +640,14 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } catch (Exception e) {} })); } + Futures.allAsList(futures).get(); pageLink = pageLink.nextPageLink(); } while (tenantIds.hasNext()); pageLink = new PageLink(1000); PageData> pairs; - Set tenants = new HashSet<>(); do { + List> futures = new ArrayList<>(); pairs = assetDao.getAllAssetTypes(pageLink); for (TbPair pair : pairs.getData()) { TenantId tenantId = new TenantId(pair.getFirst()); @@ -661,17 +660,13 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService })); } } + Futures.allAsList(futures).get(); pageLink = pageLink.nextPageLink(); } while (pairs.hasNext()); - Futures.allAsList(futures).get(); - log.info("Updating asset profiles..."); conn.createStatement().execute("call update_asset_profiles()"); - conn.createStatement().execute("UPDATE asset a SET asset_profile_id = " + - "(SELECT id FROM asset_profile ap WHERE ap.tenant_id = a.tenant_id AND name='default') WHERE a.asset_profile_id IS NULL;"); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.1", "schema_update_after.sql"); loadSql(schemaUpdateFile, conn); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java rename to common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java index 4cea485575..c7dba0c848 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbPair.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.model.sql; +package org.thingsboard.server.common.data.util; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index d17181cffa..46a1a1bac5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import java.util.List; import java.util.Optional; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index cbb68b6ccc..aec62f0021 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -23,7 +23,7 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import java.util.List; import java.util.UUID; @@ -197,7 +197,7 @@ public interface AssetRepository extends JpaRepository, Expor @Query("SELECT externalId FROM AssetEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); - @Query(value = "SELECT DISTINCT new org.thingsboard.server.dao.model.sql.TbPair(a.tenantId , a.type) FROM AssetEntity a") + @Query(value = "SELECT DISTINCT new org.thingsboard.server.common.data.util.TbPair(a.tenantId , a.type) FROM AssetEntity a") Page> getAllAssetTypes(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index a0c70428a5..b0e1921e87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -33,7 +33,7 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; -import org.thingsboard.server.dao.model.sql.TbPair; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.util.SqlDao; From 4ca140f306a026b6315f0de9f5f69cc6a26a4143 Mon Sep 17 00:00:00 2001 From: dlandiak Date: Wed, 9 Nov 2022 15:01:15 +0200 Subject: [PATCH 26/50] device state svc refactoring --- .../state/DefaultDeviceStateService.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 4aaba9c6ea..ed9949a379 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -642,6 +642,11 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService Date: Wed, 9 Nov 2022 15:30:45 +0200 Subject: [PATCH 27/50] Added RPC_REQUEST and RPC_RESPONSE to the front-end --- ui-ngx/src/app/shared/models/edge.models.ts | 10 ++++++++-- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/shared/models/edge.models.ts b/ui-ngx/src/app/shared/models/edge.models.ts index 59f9d54318..6a84b6d29c 100644 --- a/ui-ngx/src/app/shared/models/edge.models.ts +++ b/ui-ngx/src/app/shared/models/edge.models.ts @@ -77,7 +77,9 @@ export enum EdgeEventActionType { UNASSIGNED_FROM_CUSTOMER = 'UNASSIGNED_FROM_CUSTOMER', RELATION_ADD_OR_UPDATE = 'RELATION_ADD_OR_UPDATE', RELATION_DELETED = 'RELATION_DELETED', - RPC_CALL = 'RPC_CALL', + RPC_CALL = 'RPC_CALL', // deprecated - to be removed in 4.x + RPC_CALL_REQUEST = 'RPC_CALL_REQUEST', + RPC_CALL_RESPONSE = 'RPC_CALL_RESPONSE', ALARM_ACK = 'ALARM_ACK', ALARM_CLEAR = 'ALARM_CLEAR', ASSIGNED_TO_EDGE = 'ASSIGNED_TO_EDGE', @@ -128,6 +130,8 @@ export const edgeEventActionTypeTranslations = new Map( diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2b1eb11de7..c4031b6d0c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1791,6 +1791,8 @@ "action-type-relation-add-or-update": "Relation Add or Update", "action-type-relation-deleted": "Relation Deleted", "action-type-rpc-call": "RPC Call", + "action-type-rpc-call-request": "RPC Call Request", + "action-type-rpc-call-response": "RPC Call Response", "action-type-alarm-ack": "Alarm Ack", "action-type-alarm-clear": "Alarm Clear", "action-type-assigned-to-edge": "Assigned to Edge", From aefef78fda9556e4d887d2195a97cc9fc38493ef Mon Sep 17 00:00:00 2001 From: dlandiak Date: Wed, 9 Nov 2022 15:47:12 +0200 Subject: [PATCH 28/50] added defaultInactivityTimeoutMs param to device state svc --- .../service/state/DefaultDeviceStateService.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index ed9949a379..a223d97c96 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -157,6 +157,9 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService transformInactivityTimeout(ListenableFuture future) { return Futures.transformAsync(future, deviceStateData -> { - if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != TimeUnit.SECONDS.toMillis(defaultInactivityTimeoutInSec)) { + if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != defaultInactivityTimeoutMs) { return future; //fail fast } var attributesFuture = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), SERVER_SCOPE, INACTIVITY_TIMEOUT); @@ -567,7 +570,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService Date: Wed, 9 Nov 2022 15:57:18 +0200 Subject: [PATCH 29/50] fix tests in device state svc --- .../server/service/state/DefaultDeviceStateService.java | 2 ++ .../server/service/state/DefaultDeviceStateServiceTest.java | 1 + 2 files changed, 3 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index a223d97c96..d52c712074 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -158,6 +158,8 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService> latest, long defaultInactivityTimeoutInSec) { service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec); + service.setDefaultInactivityTimeoutMs(defaultInactivityTimeoutInSec * 1000); service.setPersistToTelemetry(true); var deviceUuid = UUID.randomUUID(); From 323d3d51389b53e77b88e8dfe63f03ad97e3d531 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 9 Nov 2022 16:33:43 +0200 Subject: [PATCH 30/50] fix bug for findAlarmDataByQueryForEntities method when sortOrder is null and textSearch used --- .../sql/query/DefaultAlarmQueryRepository.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 69cb4a2d86..66219eb9ba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -140,6 +140,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { selectPart.append(" a.originator_id as entity_id "); } EntityDataSortOrder sortOrder = pageLink.getSortOrder(); + String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch()); if (sortOrder != null && sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { String sortOrderKey = sortOrder.getKey().getKey(); sortPart.append(alarmFieldColumnMap.getOrDefault(sortOrderKey, sortOrderKey)) @@ -166,7 +167,11 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } joinPart.append(" as e(id, priority)) e "); if (pageLink.isSearchPropagatedAlarms()) { - joinPart.append("on ea.entity_id = e.id"); + if (textSearchQuery.isEmpty()) { + joinPart.append("on ea.entity_id = e.id"); + } else { + joinPart.append("on a.entity_id = e.id"); + } } else { joinPart.append("on a.originator_id = e.id"); } @@ -230,13 +235,11 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } } - String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch()); - String mainQuery; - if (!textSearchQuery.isEmpty()) { - mainQuery = selectPart.toString() + fromPart.toString() + wherePart.toString(); - mainQuery = String.format("select * from (%s) a %s WHERE %s", mainQuery, joinPart, textSearchQuery); + String mainQuery = String.format("%s%s", selectPart, fromPart); + if (textSearchQuery.isEmpty()) { + mainQuery += String.format("%s%s", joinPart, wherePart); } else { - mainQuery = selectPart.toString() + fromPart.toString() + joinPart.toString() + wherePart.toString(); + mainQuery = String.format("select * from (%s%s) a %s WHERE %s", mainQuery, wherePart, joinPart, textSearchQuery); } String countQuery = String.format("select count(*) from (%s) result", mainQuery); long queryTs = System.currentTimeMillis(); From 1b5d988d71d3a76c67341f8ec2e10dc887746367 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 9 Nov 2022 17:43:47 +0200 Subject: [PATCH 31/50] Improve upgrade from 3.4.1 to clear assets and repositorySettings cache --- .../service/install/update/DefaultCacheCleanupService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index 9af49f4a7c..6e67876dc4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -73,6 +73,11 @@ public class DefaultCacheCleanupService implements CacheCleanupService { log.info("Clear cache to upgrade from version 3.3.4 to 3.4.0 ..."); clearAll(); break; + case "3.4.1": + log.info("Clear cache to upgrade from version 3.4.1 to 3.4.2 ..."); + clearCacheByName("assets"); + clearCacheByName("repositorySettings"); + break; default: //Do nothing, since cache cleanup is optional. } From 385c3acaeed617d216a3501cf5567caacab9cea3 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 9 Nov 2022 17:56:38 +0200 Subject: [PATCH 32/50] UI: Fixed dashboard layout function setFixedLayout --- ...ge-dashboard-layouts-dialog.component.html | 22 ++++++++----- ...ge-dashboard-layouts-dialog.component.scss | 4 --- ...nage-dashboard-layouts-dialog.component.ts | 31 ++++++------------- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.html index 98c5226228..74e7db937b 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/layout/manage-dashboard-layouts-dialog.component.html @@ -65,12 +65,15 @@ mat-raised-button color="primary" class="tb-layout-button" - [matTooltip]="layoutButtonText('main')" - matTooltipPosition="above" - matTooltipClass="tb-layout-button-tooltip" + (mouseover)="mainLayoutTooltip.show()" + (mouseleave)="mainLayoutTooltip.hide()" (click)="setFixedLayout('main')" [ngClass]="layoutButtonClass('main', true)"> - {{ (layoutsFormGroup.value.right ? 'layout.left' : 'layout.main') | translate }} + + {{ (layoutsFormGroup.value.right ? 'layout.left' : 'layout.main') | translate }} +

- {{ 'layout.right' | translate }} + + {{ 'layout.right' | translate }} +
{ - elementToDisable.disabled = false; - }, 250); - } - - if (this.layoutsFormGroup.get('type').value === LayoutWidthType.FIXED) { + if (this.layoutsFormGroup.get('type').value === LayoutWidthType.FIXED && this.layoutsFormGroup.get('right').value) { this.layoutsFormGroup.get('fixedLayout').setValue(layout); } } From 5ff8144f8db2f5f2fe95e9bb6e0658faeab78367 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 9 Nov 2022 18:31:08 +0200 Subject: [PATCH 33/50] Code cleanup --- .../rpc/processor/DeviceEdgeProcessor.java | 16 +-------- common/edge-api/src/main/proto/edge.proto | 10 +++--- .../engine/edge/AbstractTbMsgPushNode.java | 5 ++- .../rule/engine/rpc/TbSendRPCReplyNode.java | 33 +++---------------- 4 files changed, 14 insertions(+), 50 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java index 959f5c5fe8..147bb454fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java @@ -369,36 +369,22 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { private ListenableFuture processDeviceRpcRequestFromEdge(TenantId tenantId, Edge edge, DeviceRpcCallMsg deviceRpcCallMsg) { DeviceId deviceId = new DeviceId(new UUID(deviceRpcCallMsg.getDeviceIdMSB(), deviceRpcCallMsg.getDeviceIdLSB())); - UUID requestUUID = new UUID(deviceRpcCallMsg.getRequestUuidMSB(), deviceRpcCallMsg.getRequestUuidLSB()); try { - ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.createObjectNode(); TbMsgMetaData metaData = new TbMsgMetaData(); String requestId = Integer.toString(deviceRpcCallMsg.getRequestId()); metaData.putValue("requestId", requestId); - metaData.putValue("requestUUID", requestUUID.toString()); metaData.putValue("serviceId", deviceRpcCallMsg.getServiceId()); metaData.putValue("sessionId", deviceRpcCallMsg.getSessionId()); - metaData.putValue("expirationTime", Long.toString(deviceRpcCallMsg.getExpirationTime())); - metaData.putValue("oneway", Boolean.toString(deviceRpcCallMsg.getOneway())); - metaData.putValue(DataConstants.PERSISTENT, Boolean.toString(deviceRpcCallMsg.getPersisted())); - - if (deviceRpcCallMsg.getRetries() > 0) { - metaData.putValue(DataConstants.RETRIES, Integer.toString(deviceRpcCallMsg.getRetries())); - } - metaData.putValue(DataConstants.EDGE_ID, edge.getId().toString()); - Device device = deviceService.findDeviceById(tenantId, deviceId); if (device != null) { metaData.putValue("deviceName", device.getName()); metaData.putValue("deviceType", device.getType()); metaData.putValue(DataConstants.DEVICE_ID, deviceId.getId().toString()); } - + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.createObjectNode(); entityNode.put("method", deviceRpcCallMsg.getRequestMsg().getMethod()); entityNode.put("params", deviceRpcCallMsg.getRequestMsg().getParams()); - - entityNode.put(DataConstants.ADDITIONAL_INFO, deviceRpcCallMsg.getAdditionalInfo()); TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, null, metaData, TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() { diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index d70179cfe0..249f251238 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -430,11 +430,11 @@ message DeviceRpcCallMsg { bool oneway = 7; RpcRequestMsg requestMsg = 8; RpcResponseMsg responseMsg = 9; - bool persisted = 10; - int32 retries = 11; - string additionalInfo = 12; - string serviceId = 13; - string sessionId = 14; + optional bool persisted = 10; + optional int32 retries = 11; + optional string additionalInfo = 12; + optional string serviceId = 13; + optional string sessionId = 14; } message RpcRequestMsg { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java index 28dc64c068..f120276df3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java @@ -142,8 +142,11 @@ public abstract class AbstractTbMsgPushNode future = ctx.getEdgeEventService().saveAsync(edgeEvent); Futures.addCallback(future, new FutureCallback() { @Override From 1635d883a77c41ab4714f80030bf92594bb56fc0 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 9 Nov 2022 18:32:51 +0200 Subject: [PATCH 34/50] Rename variable --- .../service/edge/rpc/processor/DeviceEdgeProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java index 147bb454fa..4a517e19e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java @@ -382,11 +382,11 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { metaData.putValue("deviceType", device.getType()); metaData.putValue(DataConstants.DEVICE_ID, deviceId.getId().toString()); } - ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.createObjectNode(); - entityNode.put("method", deviceRpcCallMsg.getRequestMsg().getMethod()); - entityNode.put("params", deviceRpcCallMsg.getRequestMsg().getParams()); + ObjectNode data = JacksonUtil.OBJECT_MAPPER.createObjectNode(); + data.put("method", deviceRpcCallMsg.getRequestMsg().getMethod()); + data.put("params", deviceRpcCallMsg.getRequestMsg().getParams()); TbMsg tbMsg = TbMsg.newMsg(SessionMsgType.TO_SERVER_RPC_REQUEST.name(), deviceId, null, metaData, - TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(data)); tbClusterService.pushMsgToRuleEngine(tenantId, deviceId, tbMsg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { From 54f967e944b20ad5ea4b2e77ac67d6d5beacc34d Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 9 Nov 2022 18:35:18 +0200 Subject: [PATCH 35/50] Remove unused file --- .../rpc/processor/EdgeRpcRequestMetadata.java | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java deleted file mode 100644 index a5a71d0743..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/EdgeRpcRequestMetadata.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edge.rpc.processor; - -import lombok.Data; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.TenantId; - -@Data -public class EdgeRpcRequestMetadata { - private final TenantId tenantId; - private final EdgeId edgeId; - private final DeviceId deviceId; -} From 0a56f065f7218472d23818ac83e66c0daba9c3d9 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 10 Nov 2022 09:35:06 +0200 Subject: [PATCH 36/50] Merge RPC request/response actions --- .../device/DeviceActorMessageProcessor.java | 2 +- .../service/edge/rpc/EdgeGrpcSession.java | 3 +- .../rpc/constructor/DeviceMsgConstructor.java | 35 +++++++------------ .../rpc/processor/DeviceEdgeProcessor.java | 21 +++-------- .../server/edge/BaseDeviceEdgeTest.java | 2 +- .../common/data/edge/EdgeEventActionType.java | 3 +- .../rule/engine/rpc/TbSendRPCReplyNode.java | 2 +- ui-ngx/src/app/shared/models/edge.models.ts | 10 ++---- .../assets/locale/locale.constant-en_US.json | 2 -- 9 files changed, 25 insertions(+), 55 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 9b95b188d2..8a365b4e56 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -827,7 +827,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { body.put("retries", msg.getRetries()); body.put("additionalInfo", msg.getAdditionalInfo()); - EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(tenantId, edgeId, EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL_REQUEST, deviceId, body); + EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(tenantId, edgeId, EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL, deviceId, body); return Futures.transform(systemContext.getEdgeEventService().saveAsync(edgeEvent), unused -> { systemContext.getClusterService().onEdgeEventUpdate(tenantId, edgeId); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index aca2f8b562..957c699e32 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -463,8 +463,7 @@ public final class EdgeGrpcSession implements Closeable { case UNASSIGNED_FROM_CUSTOMER: case CREDENTIALS_REQUEST: case ENTITY_MERGE_REQUEST: - case RPC_CALL_REQUEST: - case RPC_CALL_RESPONSE: + case RPC_CALL: downlinkMsg = convertEntityEventToDownlink(edgeEvent); log.trace("[{}][{}] entity message processed [{}]", edgeEvent.getTenantId(), this.sessionId, downlinkMsg); break; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java index ebea5d482f..522bcd3a4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceMsgConstructor.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; @@ -97,30 +96,22 @@ public class DeviceMsgConstructor { .setIdLSB(deviceId.getId().getLeastSignificantBits()).build(); } - public DeviceRpcCallMsg constructDeviceRpcRequestMsg(UUID deviceId, JsonNode body) { + public DeviceRpcCallMsg constructDeviceRpcCallMsg(UUID deviceId, JsonNode body) { DeviceRpcCallMsg.Builder builder = constructDeviceRpcMsg(deviceId, body); - - String method = body.get("method").asText(); - String params = body.get("params").asText(); - RpcRequestMsg.Builder requestBuilder = RpcRequestMsg.newBuilder(); - requestBuilder.setMethod(method); - requestBuilder.setParams(params); - builder.setRequestMsg(requestBuilder.build()); - - return builder.build(); - } - - public DeviceRpcCallMsg constructDeviceRpcResponseMsg(UUID deviceId, JsonNode body) { - DeviceRpcCallMsg.Builder builder = constructDeviceRpcMsg(deviceId, body); - - RpcResponseMsg.Builder responseBuilder = RpcResponseMsg.newBuilder(); - if (body.has("error")) { - responseBuilder.setError(body.get("error").asText()); + if (body.has("error") || body.has("response")) { + RpcResponseMsg.Builder responseBuilder = RpcResponseMsg.newBuilder(); + if (body.has("error")) { + responseBuilder.setError(body.get("error").asText()); + } else { + responseBuilder.setResponse(body.get("response").asText()); + } + builder.setResponseMsg(responseBuilder.build()); } else { - responseBuilder.setResponse(body.get("response").asText()); + RpcRequestMsg.Builder requestBuilder = RpcRequestMsg.newBuilder(); + requestBuilder.setMethod(body.get("method").asText()); + requestBuilder.setParams(body.get("params").asText()); + builder.setRequestMsg(requestBuilder.build()); } - builder.setResponseMsg(responseBuilder.build()); - return builder.build(); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java index 4a517e19e9..81e7970611 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/DeviceEdgeProcessor.java @@ -451,10 +451,8 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { .build(); } break; - case RPC_CALL_REQUEST: - return convertRpcCallRequestEventToDownlink(edgeEvent); - case RPC_CALL_RESPONSE: - return convertRpcCallResponseEventToDownlink(edgeEvent); + case RPC_CALL: + return convertRpcCallEventToDownlink(edgeEvent); case CREDENTIALS_REQUEST: return convertCredentialsRequestEventToDownlink(edgeEvent); case ENTITY_MERGE_REQUEST: @@ -463,20 +461,11 @@ public class DeviceEdgeProcessor extends BaseEdgeProcessor { return downlinkMsg; } - private DownlinkMsg convertRpcCallRequestEventToDownlink(EdgeEvent edgeEvent) { - log.trace("Executing convertRpcCallRequestEventToDownlink, edgeEvent [{}]", edgeEvent); + private DownlinkMsg convertRpcCallEventToDownlink(EdgeEvent edgeEvent) { + log.trace("Executing convertRpcCallEventToDownlink, edgeEvent [{}]", edgeEvent); return DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addDeviceRpcCallMsg(deviceMsgConstructor.constructDeviceRpcRequestMsg(edgeEvent.getEntityId(), edgeEvent.getBody())) - .build(); - } - - private DownlinkMsg convertRpcCallResponseEventToDownlink(EdgeEvent edgeEvent) { - log.trace("Executing convertRpcCallResponseEventToDownlink, edgeEvent [{}]", edgeEvent); - DeviceRpcCallMsg deviceRpcCallMsg = deviceMsgConstructor.constructDeviceRpcResponseMsg(edgeEvent.getEntityId(), edgeEvent.getBody()); - return DownlinkMsg.newBuilder() - .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) - .addDeviceRpcCallMsg(deviceRpcCallMsg) + .addDeviceRpcCallMsg(deviceMsgConstructor.constructDeviceRpcCallMsg(edgeEvent.getEntityId(), edgeEvent.getBody())) .build(); } diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java index 8e8c616d7a..be8089e4c0 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java @@ -512,7 +512,7 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest { body.put("method", "test_method"); body.put("params", "{\"param1\":\"value1\"}"); - EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.RPC_CALL_REQUEST, + EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.RPC_CALL, device.getId().getId(), EdgeEventType.DEVICE, body); edgeImitator.expectMessageAmount(1); edgeEventService.saveAsync(edgeEvent).get(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java index 7fae6c56b1..bba8767fca 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventActionType.java @@ -28,8 +28,7 @@ public enum EdgeEventActionType { UNASSIGNED_FROM_CUSTOMER, RELATION_ADD_OR_UPDATE, RELATION_DELETED, - RPC_CALL_REQUEST, - RPC_CALL_RESPONSE, + RPC_CALL, ALARM_ACK, ALARM_CLEAR, ASSIGNED_TO_EDGE, diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index cf0ebefc01..db3953f6cb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -96,7 +96,7 @@ public class TbSendRPCReplyNode implements TbNode { EdgeId edgeId = new EdgeId(UUID.fromString(msg.getMetaData().getValue(DataConstants.EDGE_ID))); DeviceId deviceId = new DeviceId(UUID.fromString(msg.getMetaData().getValue(DataConstants.DEVICE_ID))); EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(ctx.getTenantId(), edgeId, EdgeEventType.DEVICE, - EdgeEventActionType.RPC_CALL_RESPONSE, deviceId, JacksonUtil.OBJECT_MAPPER.valueToTree(body)); + EdgeEventActionType.RPC_CALL, deviceId, JacksonUtil.OBJECT_MAPPER.valueToTree(body)); ListenableFuture future = ctx.getEdgeEventService().saveAsync(edgeEvent); Futures.addCallback(future, new FutureCallback() { @Override diff --git a/ui-ngx/src/app/shared/models/edge.models.ts b/ui-ngx/src/app/shared/models/edge.models.ts index 6a84b6d29c..59f9d54318 100644 --- a/ui-ngx/src/app/shared/models/edge.models.ts +++ b/ui-ngx/src/app/shared/models/edge.models.ts @@ -77,9 +77,7 @@ export enum EdgeEventActionType { UNASSIGNED_FROM_CUSTOMER = 'UNASSIGNED_FROM_CUSTOMER', RELATION_ADD_OR_UPDATE = 'RELATION_ADD_OR_UPDATE', RELATION_DELETED = 'RELATION_DELETED', - RPC_CALL = 'RPC_CALL', // deprecated - to be removed in 4.x - RPC_CALL_REQUEST = 'RPC_CALL_REQUEST', - RPC_CALL_RESPONSE = 'RPC_CALL_RESPONSE', + RPC_CALL = 'RPC_CALL', ALARM_ACK = 'ALARM_ACK', ALARM_CLEAR = 'ALARM_CLEAR', ASSIGNED_TO_EDGE = 'ASSIGNED_TO_EDGE', @@ -130,8 +128,6 @@ export const edgeEventActionTypeTranslations = new Map( diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 17b4533b03..3f3f70860f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1791,8 +1791,6 @@ "action-type-relation-add-or-update": "Relation Add or Update", "action-type-relation-deleted": "Relation Deleted", "action-type-rpc-call": "RPC Call", - "action-type-rpc-call-request": "RPC Call Request", - "action-type-rpc-call-response": "RPC Call Response", "action-type-alarm-ack": "Alarm Ack", "action-type-alarm-clear": "Alarm Clear", "action-type-assigned-to-edge": "Assigned to Edge", From 15b26f4317f8675b73bd2cfc0ff55336c633daf1 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 10 Nov 2022 10:37:14 +0200 Subject: [PATCH 37/50] Remove Nullable annotation --- .../org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index db3953f6cb..796aa5658c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; @@ -98,9 +97,9 @@ public class TbSendRPCReplyNode implements TbNode { EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(ctx.getTenantId(), edgeId, EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL, deviceId, JacksonUtil.OBJECT_MAPPER.valueToTree(body)); ListenableFuture future = ctx.getEdgeEventService().saveAsync(edgeEvent); - Futures.addCallback(future, new FutureCallback() { + Futures.addCallback(future, new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void result) { + public void onSuccess(Void result) { ctx.onEdgeEventUpdate(ctx.getTenantId(), edgeId); ctx.tellSuccess(msg); } From d8a609344289f9ae7de77712cbbd1793b1a912a1 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 10 Nov 2022 11:36:46 +0200 Subject: [PATCH 38/50] Added TbSendRPCReplyNodeTest --- .../server/edge/BaseDeviceEdgeTest.java | 4 + .../rule/engine/rpc/TbSendRPCReplyNode.java | 13 +- .../engine/rpc/TbSendRPCReplyNodeTest.java | 119 ++++++++++++++++++ 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java index be8089e4c0..ffe21ae961 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java @@ -511,6 +511,8 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest { body.put("expirationTime", System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10)); body.put("method", "test_method"); body.put("params", "{\"param1\":\"value1\"}"); + body.put("persisted", true); + body.put("retries", 2); EdgeEvent edgeEvent = constructEdgeEvent(tenantId, edge.getId(), EdgeEventActionType.RPC_CALL, device.getId().getId(), EdgeEventType.DEVICE, body); @@ -523,6 +525,8 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest { Assert.assertTrue(latestMessage instanceof DeviceRpcCallMsg); DeviceRpcCallMsg latestDeviceRpcCallMsg = (DeviceRpcCallMsg) latestMessage; Assert.assertEquals("test_method", latestDeviceRpcCallMsg.getRequestMsg().getMethod()); + Assert.assertTrue(latestDeviceRpcCallMsg.getPersisted()); + Assert.assertEquals(2, latestDeviceRpcCallMsg.getRetries()); } private void sendAttributesRequestAndVerify(Device device, String scope, String attributesDataStr, String expectedKey, diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java index 796aa5658c..405a73944e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNode.java @@ -87,13 +87,22 @@ public class TbSendRPCReplyNode implements TbNode { } private void saveRpcResponseToEdgeQueue(TbContext ctx, TbMsg msg, String serviceIdStr, String sessionIdStr, String requestIdStr) { + EdgeId edgeId; + DeviceId deviceId; + try { + edgeId = new EdgeId(UUID.fromString(msg.getMetaData().getValue(DataConstants.EDGE_ID))); + deviceId = new DeviceId(UUID.fromString(msg.getMetaData().getValue(DataConstants.DEVICE_ID))); + } catch (Exception e) { + String errMsg = String.format("[%s] Failed to parse edgeId or deviceId from metadata %s!", ctx.getTenantId(), msg.getMetaData()); + ctx.tellFailure(msg, new RuntimeException(errMsg)); + return; + } + ObjectNode body = JacksonUtil.OBJECT_MAPPER.createObjectNode(); body.put("serviceId", serviceIdStr); body.put("sessionId", sessionIdStr); body.put("requestId", requestIdStr); body.put("response", msg.getData()); - EdgeId edgeId = new EdgeId(UUID.fromString(msg.getMetaData().getValue(DataConstants.EDGE_ID))); - DeviceId deviceId = new DeviceId(UUID.fromString(msg.getMetaData().getValue(DataConstants.DEVICE_ID))); EdgeEvent edgeEvent = EdgeUtils.constructEdgeEvent(ctx.getTenantId(), edgeId, EdgeEventType.DEVICE, EdgeEventActionType.RPC_CALL, deviceId, JacksonUtil.OBJECT_MAPPER.valueToTree(body)); ListenableFuture future = ctx.getEdgeEventService().saveAsync(edgeEvent); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java new file mode 100644 index 0000000000..095505aff3 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.rpc; + +import com.google.common.util.concurrent.SettableFuture; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.rule.engine.api.RuleEngineRpcService; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.dao.edge.EdgeEventService; + +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +@RunWith(MockitoJUnitRunner.class) +public class TbSendRPCReplyNodeTest { + + private static final String DUMMY_SERVICE_ID = "testServiceId"; + private static final int DUMMY_REQUEST_ID = 0; + private static final UUID DUMMY_SESSION_ID = UUID.randomUUID(); + private static final String DUMMY_DATA = "{\"key\":\"value\"}"; + + TbSendRPCReplyNode node; + + private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + private final DeviceId deviceId = new DeviceId(UUID.randomUUID()); + + @Mock + private TbContext ctx; + + @Mock + private RuleEngineRpcService rpcService; + + @Mock + private EdgeEventService edgeEventService; + + @Mock + private ListeningExecutor listeningExecutor; + + @Before + public void setUp() throws TbNodeException { + node = new TbSendRPCReplyNode(); + TbSendRpcReplyNodeConfiguration config = new TbSendRpcReplyNodeConfiguration().defaultConfiguration(); + node.init(ctx, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + } + + @Test + public void sendReplyToTransport() { + Mockito.when(ctx.getRpcService()).thenReturn(rpcService); + + + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, getDefaultMetadata(), + TbMsgDataType.JSON, DUMMY_DATA, null, null); + + node.onMsg(ctx, msg); + + verify(rpcService).sendRpcReplyToDevice(DUMMY_SERVICE_ID, DUMMY_SESSION_ID, DUMMY_REQUEST_ID, DUMMY_DATA); + verify(edgeEventService, never()).saveAsync(any()); + } + + @Test + public void sendReplyToEdgeQueue() { + Mockito.when(ctx.getTenantId()).thenReturn(tenantId); + Mockito.when(ctx.getEdgeEventService()).thenReturn(edgeEventService); + Mockito.when(edgeEventService.saveAsync(any())).thenReturn(SettableFuture.create()); + Mockito.when(ctx.getDbCallbackExecutor()).thenReturn(listeningExecutor); + + TbMsgMetaData defaultMetadata = getDefaultMetadata(); + defaultMetadata.putValue(DataConstants.EDGE_ID, UUID.randomUUID().toString()); + defaultMetadata.putValue(DataConstants.DEVICE_ID, UUID.randomUUID().toString()); + TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, defaultMetadata, + TbMsgDataType.JSON, DUMMY_DATA, null, null); + + node.onMsg(ctx, msg); + + verify(edgeEventService).saveAsync(any()); + verify(rpcService, never()).sendRpcReplyToDevice(DUMMY_SERVICE_ID, DUMMY_SESSION_ID, DUMMY_REQUEST_ID, DUMMY_DATA); + } + + private TbMsgMetaData getDefaultMetadata() { + TbSendRpcReplyNodeConfiguration config = new TbSendRpcReplyNodeConfiguration().defaultConfiguration(); + TbMsgMetaData metadata = new TbMsgMetaData(); + metadata.putValue(config.getServiceIdMetaDataAttribute(), DUMMY_SERVICE_ID); + metadata.putValue(config.getSessionIdMetaDataAttribute(), DUMMY_SESSION_ID.toString()); + metadata.putValue(config.getRequestIdMetaDataAttribute(), Integer.toString(DUMMY_REQUEST_ID)); + return metadata; + } +} From 6223307cd8eb7e5ab7d5c022762a2926ef658f75 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 10 Nov 2022 17:04:48 +0200 Subject: [PATCH 39/50] Add validation for Asset Profile FK in Rule Chains --- .../org/thingsboard/server/dao/rule/BaseRuleChainService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 420739e37c..1d11d34b2d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -731,6 +731,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("fk_default_rule_chain_device_profile")) { throw new DataValidationException("The rule chain referenced by the device profiles cannot be deleted!"); + } else if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("fk_default_rule_chain_asset_profile")) { + throw new DataValidationException("The rule chain referenced by the asset profiles cannot be deleted!"); } else { throw t; } From 7b9c9e51cd165033a7654b1994daba3dc2bf2c04 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Fri, 11 Nov 2022 14:41:15 +0200 Subject: [PATCH 40/50] update build of main query for alarm count query --- .../server/dao/sql/query/DefaultAlarmQueryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 66219eb9ba..d91a8c5fc4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -237,7 +237,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { String mainQuery = String.format("%s%s", selectPart, fromPart); if (textSearchQuery.isEmpty()) { - mainQuery += String.format("%s%s", joinPart, wherePart); + mainQuery = String.format("%s%s%s", mainQuery, joinPart, wherePart); } else { mainQuery = String.format("select * from (%s%s) a %s WHERE %s", mainQuery, wherePart, joinPart, textSearchQuery); } From 8a29dc294e35f130d88438736f13c71a02f9d5b8 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 11 Nov 2022 17:41:12 +0200 Subject: [PATCH 41/50] mute TelemetryEdgeSqlTest that causes a lot of randomly generated errors --- application/src/test/resources/logback-test.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index d3301bf660..3762c8aa7c 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -16,6 +16,8 @@ + + From e32bd456b7875555f0b0b0654288f297e368fa0f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 14 Nov 2022 11:07:09 +0100 Subject: [PATCH 42/50] added memory usage log to the js-executors --- msa/js-executor/api/jsInvokeMessageProcessor.ts | 5 +++++ msa/js-executor/config/custom-environment-variables.yml | 1 + msa/js-executor/config/default.yml | 1 + 3 files changed, 7 insertions(+) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.ts b/msa/js-executor/api/jsInvokeMessageProcessor.ts index 668cd61f50..e69209fa01 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.ts +++ b/msa/js-executor/api/jsInvokeMessageProcessor.ts @@ -39,6 +39,7 @@ const TIMEOUT_ERROR = 2; const NOT_FOUND_ERROR = 3; const statFrequency = Number(config.get('script.stat_print_frequency')); +const memoryUsageTraceFrequency = Number(config.get('script.memory_usage_trace_frequency')); const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency')); const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); @@ -167,6 +168,10 @@ export class JsInvokeMessageProcessor { if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) { this.logger.info('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody); } + if (this.executedScriptsCounter % memoryUsageTraceFrequency == 0) { + this.logger.info('Current memory usage: [%s]', process.memoryUsage()); + } + this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then( (script) => { this.executor.executeScript(script, invokeRequest.args, invokeRequest.timeout).then( diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index b9c24c8d8d..2ebea4ccc1 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -75,6 +75,7 @@ logger: script: use_sandbox: "SCRIPT_USE_SANDBOX" + memory_usage_trace_frequency: "MEMORY_USAGE_TRACE_FREQUENCY" stat_print_frequency: "SCRIPT_STAT_PRINT_FREQUENCY" script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY" max_active_scripts: "MAX_ACTIVE_SCRIPTS" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 64829ef792..805c175dce 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -64,6 +64,7 @@ logger: script: use_sandbox: "true" + memory_usage_trace_frequency: "10000" script_body_trace_frequency: "10000" stat_print_frequency: "10000" max_active_scripts: "1000" From 11bd9b5257344b60c7e5eb1a2bbe6af175ffde5b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 14 Nov 2022 11:28:30 +0100 Subject: [PATCH 43/50] added nodejs memory leak workaround --- docker/tb-js-executor.env | 3 ++- msa/js-executor/config/default.yml | 2 +- msa/js-executor/docker/start-js-executor.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index e080906549..1938449d53 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -3,4 +3,5 @@ LOGGER_LEVEL=info LOG_FOLDER=logs LOGGER_FILENAME=tb-js-executor-%DATE%.log DOCKER_MODE=true -SCRIPT_BODY_TRACE_FREQUENCY=1000 \ No newline at end of file +SCRIPT_BODY_TRACE_FREQUENCY=1000 +NODE_OPTIONS="--max-old-space-size=200" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 805c175dce..96f3401da5 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -64,7 +64,7 @@ logger: script: use_sandbox: "true" - memory_usage_trace_frequency: "10000" + memory_usage_trace_frequency: "1000" script_body_trace_frequency: "10000" stat_print_frequency: "10000" max_active_scripts: "1000" diff --git a/msa/js-executor/docker/start-js-executor.sh b/msa/js-executor/docker/start-js-executor.sh index 575f93c389..b27c1b7167 100755 --- a/msa/js-executor/docker/start-js-executor.sh +++ b/msa/js-executor/docker/start-js-executor.sh @@ -27,4 +27,4 @@ source "${CONF_FOLDER}/${configfile}" cd ${pkg.installFolder} # This will forward this PID 1 to the node.js and forward SIGTERM for graceful shutdown as well -exec node server.js +exec --no-compilation-cache node server.js From b02a2215db8f01b41a18049790d411ba5a3be647 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 14 Nov 2022 12:33:17 +0200 Subject: [PATCH 44/50] Update MVEL version to 2.4.24TB --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 60e8783bca..e7e75c32f8 100755 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 3.5.5 3.21.9 1.42.1 - 2.4.23TB + 2.4.24TB 1.18.18 1.2.4 4.1.75.Final From 5b1642246d722a67869ee4bb8ecca511cc138608 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 14 Nov 2022 13:30:05 +0100 Subject: [PATCH 45/50] [3.4.2] fix start js (#7614) * refactoring * fixed start js-executor typo --- msa/js-executor/docker/start-js-executor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/js-executor/docker/start-js-executor.sh b/msa/js-executor/docker/start-js-executor.sh index b27c1b7167..d30b62c145 100755 --- a/msa/js-executor/docker/start-js-executor.sh +++ b/msa/js-executor/docker/start-js-executor.sh @@ -27,4 +27,4 @@ source "${CONF_FOLDER}/${configfile}" cd ${pkg.installFolder} # This will forward this PID 1 to the node.js and forward SIGTERM for graceful shutdown as well -exec --no-compilation-cache node server.js +exec node --no-compilation-cache server.js From 51ec17d9d1782b755bd8aea10eeac1897c050166 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 14 Nov 2022 14:41:10 +0200 Subject: [PATCH 46/50] Update MVEL version to 2.4.25TB --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e7e75c32f8..5f5aa3703b 100755 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 3.5.5 3.21.9 1.42.1 - 2.4.24TB + 2.4.25TB 1.18.18 1.2.4 4.1.75.Final From 25f8ff2aefed66c17adb558546cf7bc575c31257 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 14 Nov 2022 15:37:54 +0200 Subject: [PATCH 47/50] Handle undefined script execution result. --- msa/js-executor/api/jsExecutor.models.ts | 2 +- msa/js-executor/api/jsInvokeMessageProcessor.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/msa/js-executor/api/jsExecutor.models.ts b/msa/js-executor/api/jsExecutor.models.ts index 7a6b53cd8a..17407f4d50 100644 --- a/msa/js-executor/api/jsExecutor.models.ts +++ b/msa/js-executor/api/jsExecutor.models.ts @@ -56,7 +56,7 @@ export interface JsCompileResponse extends TbMessage { export interface JsInvokeResponse { success: boolean; - result: string; + result?: string; errorCode?: number; errorDetails?: string; } diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.ts b/msa/js-executor/api/jsInvokeMessageProcessor.ts index e69209fa01..52a337c74a 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.ts +++ b/msa/js-executor/api/jsInvokeMessageProcessor.ts @@ -175,8 +175,8 @@ export class JsInvokeMessageProcessor { this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then( (script) => { this.executor.executeScript(script, invokeRequest.args, invokeRequest.timeout).then( - (result) => { - if (result.length <= maxResultSize) { + (result: string | undefined) => { + if (!result || result.length <= maxResultSize) { const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse(result, true); this.logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId); this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse); @@ -328,7 +328,7 @@ export class JsInvokeMessageProcessor { } } - private static createInvokeResponse(result: string, success: boolean, errorCode?: number, err?: any): JsInvokeResponse { + private static createInvokeResponse(result: string | undefined, success: boolean, errorCode?: number, err?: any): JsInvokeResponse { return { errorCode: errorCode, success: success, From dee81fea2653ab22aa9ae97c7b9dd33c1cb0b60a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 14 Nov 2022 18:00:19 +0200 Subject: [PATCH 48/50] MVEL scripts caching --- .../src/main/resources/thingsboard.yml | 1 + .../service/script/MvelInvokeServiceTest.java | 98 +++++++++++++++++++ common/script/script-api/pom.xml | 4 + .../api/mvel/DefaultMvelInvokeService.java | 65 ++++++++++-- .../script/api/mvel/MvelScript.java | 1 - 5 files changed, 158 insertions(+), 11 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0f110da90c..cd05c7ff23 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -625,6 +625,7 @@ mvel: max_black_list_duration_sec: "${MVEL_MAX_BLACKLIST_DURATION_SEC:60}" # Specify thread pool size for javascript executor service thread_pool_size: "${MVEL_THREAD_POOL_SIZE:50}" + compiled_scripts_cache_size: "${MVEL_COMPILED_SCRIPTS_CACHE_SIZE:2000}" stats: enabled: "${TB_MVEL_STATS_ENABLED:false}" print_interval_ms: "${TB_MVEL_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java index a40a087514..1c3c134a82 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.script; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.benmanes.caffeine.cache.Cache; import org.junit.Assert; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -24,15 +25,22 @@ import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.mvel.MvelInvokeService; +import org.thingsboard.script.api.mvel.MvelScript; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest @@ -41,6 +49,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; "mvel.max_total_args_size=50", "mvel.max_result_size=50", "mvel.max_errors=2", + "mvel.compiled_scripts_cache_size=100" }) class MvelInvokeServiceTest extends AbstractControllerTest { @@ -110,6 +119,89 @@ class MvelInvokeServiceTest extends AbstractControllerTest { assertThatScriptIsBlocked(scriptId); } + @Test + void givenScriptsWithSameBody_thenCompileAndCacheOnlyOnce() throws Exception { + String script = "return msg.temperature > 20;"; + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + } + + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Map scriptMap = getFieldValue(invokeService, "scriptMap"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + String scriptHash = scriptIdToHash.get(scriptsIds.get(0)); + + assertThat(scriptsIds.stream().map(scriptIdToHash::get)).containsOnly(scriptHash); + assertThat(scriptMap).containsKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNotNull(); + } + + @Test + public void whenReleasingScript_thenCheckForScriptHashUsages() throws Exception { + String script = "return msg.temperature > 20;"; + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + } + + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Map scriptMap = getFieldValue(invokeService, "scriptMap"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + String scriptHash = scriptIdToHash.get(scriptsIds.get(0)); + for (int i = 0; i < 9; i++) { + UUID scriptId = scriptsIds.get(i); + assertThat(scriptIdToHash).containsKey(scriptId); + invokeService.release(scriptId); + assertThat(scriptIdToHash).doesNotContainKey(scriptId); + } + assertThat(scriptMap).containsKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNotNull(); + + invokeService.release(scriptsIds.get(9)); + assertThat(scriptMap).doesNotContainKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNull(); + } + + @Test + public void whenCompiledScriptsCacheIsTooBig_thenRemoveRarelyUsedScripts() throws Exception { + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 110; i++) { // mvel.compiled_scripts_cache_size = 100 + String script = "return msg.temperature > " + i; + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + + for (int j = 0; j < i; j++) { + invokeScript(scriptId, "{ \"temperature\": 12 }"); // so that scriptsIds is ordered by number of invocations + } + } + + ConcurrentMap cache = compiledScriptsCache.asMap(); + + for (int i = 0; i < 10; i++) { // iterating rarely used scripts + UUID scriptId = scriptsIds.get(i); + String scriptHash = scriptIdToHash.get(scriptId); + assertThat(cache).doesNotContainKey(scriptHash); + } + for (int i = 10; i < 110; i++) { + UUID scriptId = scriptsIds.get(i); + String scriptHash = scriptIdToHash.get(scriptId); + assertThat(cache).containsKey(scriptHash); + } + + UUID scriptRemovedFromCache = scriptsIds.get(0); + assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNull(); + invokeScript(scriptRemovedFromCache, "{ \"temperature\": 12 }"); + assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNotNull(); + } + private void assertThatScriptIsBlocked(UUID scriptId) { assertThatThrownBy(() -> { invokeScript(scriptId, "{}"); @@ -125,4 +217,10 @@ class MvelInvokeServiceTest extends AbstractControllerTest { return invokeService.invokeScript(TenantId.SYS_TENANT_ID, null, scriptId, msg, "{}", "POST_TELEMETRY_REQUEST").get().toString(); } + private T getFieldValue(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(target); + } + } diff --git a/common/script/script-api/pom.xml b/common/script/script-api/pom.xml index 62dc00ca97..b7ae4993f9 100644 --- a/common/script/script-api/pom.xml +++ b/common/script/script-api/pom.xml @@ -56,6 +56,10 @@ com.google.code.gson gson + + com.github.ben-manes.caffeine + caffeine + org.slf4j slf4j-api diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java index 901a49180e..120be2af4f 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java @@ -15,6 +15,10 @@ */ package org.thingsboard.script.api.mvel; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -42,6 +46,7 @@ import org.thingsboard.server.common.stats.TbApiUsageStateClient; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; import java.util.Optional; @@ -55,7 +60,10 @@ import java.util.regex.Pattern; @Service public class DefaultMvelInvokeService extends AbstractScriptInvokeService implements MvelInvokeService { - protected Map scriptMap = new ConcurrentHashMap<>(); + protected final Map scriptIdToHash = new ConcurrentHashMap<>(); + protected final Map scriptMap = new ConcurrentHashMap<>(); + protected Cache compiledScriptsCache; + private SandboxedParserConfiguration parserConfig; private static final Pattern NEW_KEYWORD_PATTERN = Pattern.compile("new\\s"); @@ -92,6 +100,9 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Value("${mvel.max_memory_limit_mb:8}") private long maxMemoryLimitMb; + @Value("${mvel.compiled_scripts_cache_size:2000}") + private int compiledScriptsCacheSize; + private ListeningExecutorService executor; protected DefaultMvelInvokeService(Optional apiUsageStateClient, Optional apiUsageReportClient) { @@ -115,11 +126,14 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "mvel-executor")); try { // Special command to warm up MVEL engine - Serializable script = MVEL.compileExpression("var warmUp = {}; warmUp", new SandboxedParserContext(parserConfig)); + Serializable script = compileScript("var warmUp = {}; warmUp"); MVEL.executeTbExpression(script, new ExecutionContext(parserConfig), Collections.emptyMap()); } catch (Exception e) { // do nothing } + compiledScriptsCache = Caffeine.newBuilder() + .maximumSize(compiledScriptsCacheSize) + .build(); } @PreDestroy @@ -141,16 +155,21 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Override protected boolean isScriptPresent(UUID scriptId) { - return scriptMap.containsKey(scriptId); + return scriptIdToHash.containsKey(scriptId); } @Override protected ListenableFuture doEvalScript(TenantId tenantId, ScriptType scriptType, String scriptBody, UUID scriptId, String[] argNames) { return executor.submit(() -> { try { - Serializable compiledScript = MVEL.compileExpression(scriptBody, new SandboxedParserContext(parserConfig)); - MvelScript script = new MvelScript(compiledScript, scriptBody, argNames); - scriptMap.put(scriptId, script); + String scriptHash = hash(scriptBody, argNames); + compiledScriptsCache.get(scriptHash, k -> { + return compileScript(scriptBody); + }); + scriptIdToHash.put(scriptId, scriptHash); + scriptMap.computeIfAbsent(scriptHash, k -> { + return new MvelScript(scriptBody, argNames); + }); return scriptId; } catch (Exception e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.COMPILATION, scriptBody, e); @@ -162,12 +181,16 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem protected MvelScriptExecutionTask doInvokeFunction(UUID scriptId, Object[] args) { ExecutionContext executionContext = new ExecutionContext(this.parserConfig, maxMemoryLimitMb * 1024 * 1024); return new MvelScriptExecutionTask(executionContext, executor.submit(() -> { - MvelScript script = scriptMap.get(scriptId); - if (script == null) { + String scriptHash = scriptIdToHash.get(scriptId); + if (scriptHash == null) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.OTHER, null, new RuntimeException("Script not found!")); } + MvelScript script = scriptMap.get(scriptHash); + Serializable compiledScript = compiledScriptsCache.get(scriptHash, k -> { + return compileScript(script.getScriptBody()); + }); try { - return MVEL.executeTbExpression(script.getCompiledScript(), executionContext, script.createVars(args)); + return MVEL.executeTbExpression(compiledScript, executionContext, script.createVars(args)); } catch (ScriptMemoryOverflowException e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.OTHER, script.getScriptBody(), new RuntimeException("Script memory overflow!")); } catch (Exception e) { @@ -178,6 +201,28 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Override protected void doRelease(UUID scriptId) throws Exception { - scriptMap.remove(scriptId); + String scriptHash = scriptIdToHash.remove(scriptId); + if (scriptHash != null) { + if (scriptIdToHash.containsValue(scriptHash)) { + return; + } + scriptMap.remove(scriptHash); + compiledScriptsCache.invalidate(scriptHash); + } + } + + private Serializable compileScript(String scriptBody) { + return MVEL.compileExpression(scriptBody, new SandboxedParserContext(parserConfig)); } + + @SuppressWarnings("UnstableApiUsage") + protected String hash(String scriptBody, String[] argNames) { + Hasher hasher = Hashing.murmur3_128().newHasher(); + hasher.putUnencodedChars(scriptBody); + for (String argName : argNames) { + hasher.putString(argName, StandardCharsets.UTF_8); + } + return hasher.hash().toString(); + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java index 7a84c7b0af..bca5d8d546 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java @@ -24,7 +24,6 @@ import java.util.Map; @Data public class MvelScript { - private final Serializable compiledScript; private final String scriptBody; private final String[] argNames; From 12830936815a5fed8c141fb771a44265378fe94f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 14 Nov 2022 19:02:52 +0200 Subject: [PATCH 49/50] Synchronize doEvalScript and doRelease operations in MvelInvokeService --- .../src/main/resources/thingsboard.yml | 2 +- .../api/mvel/DefaultMvelInvokeService.java | 31 +++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cd05c7ff23..28149246c9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -625,7 +625,7 @@ mvel: max_black_list_duration_sec: "${MVEL_MAX_BLACKLIST_DURATION_SEC:60}" # Specify thread pool size for javascript executor service thread_pool_size: "${MVEL_THREAD_POOL_SIZE:50}" - compiled_scripts_cache_size: "${MVEL_COMPILED_SCRIPTS_CACHE_SIZE:2000}" + compiled_scripts_cache_size: "${MVEL_COMPILED_SCRIPTS_CACHE_SIZE:1000}" stats: enabled: "${TB_MVEL_STATS_ENABLED:false}" print_interval_ms: "${TB_MVEL_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java index 120be2af4f..4fb029096b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java @@ -53,6 +53,8 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; @Slf4j @@ -100,11 +102,13 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Value("${mvel.max_memory_limit_mb:8}") private long maxMemoryLimitMb; - @Value("${mvel.compiled_scripts_cache_size:2000}") + @Value("${mvel.compiled_scripts_cache_size:1000}") private int compiledScriptsCacheSize; private ListeningExecutorService executor; + private final Lock lock = new ReentrantLock(); + protected DefaultMvelInvokeService(Optional apiUsageStateClient, Optional apiUsageReportClient) { super(apiUsageStateClient, apiUsageReportClient); } @@ -166,10 +170,15 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem compiledScriptsCache.get(scriptHash, k -> { return compileScript(scriptBody); }); - scriptIdToHash.put(scriptId, scriptHash); - scriptMap.computeIfAbsent(scriptHash, k -> { - return new MvelScript(scriptBody, argNames); - }); + lock.lock(); + try { + scriptIdToHash.put(scriptId, scriptHash); + scriptMap.computeIfAbsent(scriptHash, k -> { + return new MvelScript(scriptBody, argNames); + }); + } finally { + lock.unlock(); + } return scriptId; } catch (Exception e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.COMPILATION, scriptBody, e); @@ -203,11 +212,15 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem protected void doRelease(UUID scriptId) throws Exception { String scriptHash = scriptIdToHash.remove(scriptId); if (scriptHash != null) { - if (scriptIdToHash.containsValue(scriptHash)) { - return; + lock.lock(); + try { + if (!scriptIdToHash.containsValue(scriptHash)) { + scriptMap.remove(scriptHash); + compiledScriptsCache.invalidate(scriptHash); + } + } finally { + lock.unlock(); } - scriptMap.remove(scriptHash); - compiledScriptsCache.invalidate(scriptHash); } } From 8566ca0e946194c49fa17dbbdff37fed1612b250 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 15 Nov 2022 11:52:54 +0200 Subject: [PATCH 50/50] UI: Fixes deleting an attribute or telemetry whose name has special characters --- ui-ngx/src/app/core/http/attribute.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/core/http/attribute.service.ts b/ui-ngx/src/app/core/http/attribute.service.ts index bad6547b5d..984b1e607b 100644 --- a/ui-ngx/src/app/core/http/attribute.service.ts +++ b/ui-ngx/src/app/core/http/attribute.service.ts @@ -43,7 +43,7 @@ export class AttributeService { public deleteEntityAttributes(entityId: EntityId, attributeScope: AttributeScope, attributes: Array, config?: RequestConfig): Observable { - const keys = attributes.map(attribute => encodeURI(attribute.key)).join(','); + const keys = attributes.map(attribute => encodeURIComponent(attribute.key)).join(','); return this.http.delete(`/api/plugins/telemetry/${entityId.entityType}/${entityId.id}/${attributeScope}` + `?keys=${keys}`, defaultHttpOptionsFromConfig(config)); @@ -51,7 +51,7 @@ export class AttributeService { public deleteEntityTimeseries(entityId: EntityId, timeseries: Array, deleteAllDataForKeys = false, startTs?: number, endTs?: number, config?: RequestConfig): Observable { - const keys = timeseries.map(attribute => encodeURI(attribute.key)).join(','); + const keys = timeseries.map(attribute => encodeURIComponent(attribute.key)).join(','); let url = `/api/plugins/telemetry/${entityId.entityType}/${entityId.id}/timeseries/delete` + `?keys=${keys}&deleteAllDataForKeys=${deleteAllDataForKeys}`; if (isDefinedAndNotNull(startTs)) {