From d9b77251133551e4b8f933fe0b9a268d06730877 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 26 Mar 2024 14:24:18 +0100 Subject: [PATCH 01/42] WIP Redis junit5 test container --- dao/pom.xml | 5 ++ .../server/dao/RedisJUnit5Test.java | 64 +++++++++++++++++++ pom.xml | 12 ++++ 3 files changed, 81 insertions(+) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/RedisJUnit5Test.java diff --git a/dao/pom.xml b/dao/pom.xml index ab62d8f013..700d2b0135 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -216,6 +216,11 @@ jdbc test + + org.testcontainers + junit-jupiter + test + org.springframework spring-context-support diff --git a/dao/src/test/java/org/thingsboard/server/dao/RedisJUnit5Test.java b/dao/src/test/java/org/thingsboard/server/dao/RedisJUnit5Test.java new file mode 100644 index 0000000000..43e788cccb --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/RedisJUnit5Test.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@Testcontainers +@Slf4j +public class RedisJUnit5Test { + + @Container + private static final GenericContainer REDIS = new GenericContainer("redis:7.2-bookworm") + .withLogConsumer(s -> log.error(((OutputFrame) s).getUtf8String().trim())) + .withExposedPorts(6379); + + @BeforeAll + static void beforeAll() { + log.warn("Starting redis..."); + REDIS.start(); + System.setProperty("cache.type", "redis"); + System.setProperty("redis.connection.type", "standalone"); + System.setProperty("redis.standalone.host", REDIS.getHost()); + System.setProperty("redis.standalone.port", String.valueOf(REDIS.getMappedPort(6379))); + + } + + @AfterAll + static void afterAll() { + List.of("cache.type", "redis.connection.type", "redis.standalone.host", "redis.standalone.port") + .forEach(System.getProperties()::remove); + REDIS.stop(); + log.warn("Redis is stopped"); + } + + @Test + void test() { + assertThat(REDIS.isRunning()).isTrue(); + } + +} diff --git a/pom.xml b/pom.xml index d2450e0a65..70a35d77d2 100755 --- a/pom.xml +++ b/pom.xml @@ -2021,6 +2021,18 @@ + + org.testcontainers + junit-jupiter + ${testcontainers.version} + test + + + junit + junit + + + org.zeroturnaround zt-exec From 3a86913e24068f020ed7f274c09bd483b6cd39d5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 3 Apr 2024 10:05:01 +0200 Subject: [PATCH 02/42] RedisTbTransactionalCache refactored deprecated methods --- .../cache/RedisTbTransactionalCache.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index abfbb398c9..cd5db13995 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -29,7 +29,6 @@ import org.springframework.data.redis.core.types.Expiration; import org.springframework.data.redis.serializer.RedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; import org.thingsboard.server.common.data.FstStatsService; -import redis.clients.jedis.Connection; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.util.JedisClusterCRC16; @@ -79,7 +78,7 @@ public abstract class RedisTbTransactionalCache get(K key) { try (var connection = connectionFactory.getConnection()) { byte[] rawKey = getRawKey(key); - byte[] rawValue = connection.get(rawKey); + byte[] rawValue = connection.stringCommands().get(rawKey); if (rawValue == null) { return null; } else if (Arrays.equals(rawValue, BINARY_NULL_VALUE)) { @@ -113,7 +112,7 @@ public abstract class RedisTbTransactionalCache(this, connection); } - private RedisConnection getConnection(byte[] rawKey) { + protected RedisConnection getConnection(byte[] rawKey) { if (!connectionFactory.isRedisClusterAware()) { return connectionFactory.getConnection(); } @@ -168,7 +167,7 @@ public abstract class RedisTbTransactionalCache Date: Wed, 3 Apr 2024 11:46:03 +0200 Subject: [PATCH 03/42] WIP CachedRedisSqlTimeseriesLatestDao.java --- .../src/main/resources/thingsboard.yml | 7 ++ build.sh | 57 ++++++++++ .../server/cache/TbJavaRedisSerializer.java | 35 ++++++ .../util/SqlTsLatestAnyDaoCachedRedis.java | 26 +++++ .../server/common/data/CacheConstants.java | 1 + .../CachedRedisSqlTimeseriesLatestDao.java | 104 ++++++++++++++++++ .../dao/timeseries/TsLatestCacheKey.java | 40 +++++++ .../dao/timeseries/TsLatestRedisCache.java | 35 ++++++ .../resources/application-test.properties | 4 + dao/src/test/resources/logback.xml | 1 + 10 files changed, 310 insertions(+) create mode 100755 build.sh create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/TbJavaRedisSerializer.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDaoCachedRedis.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestCacheKey.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index daf9f5d1c1..535a396762 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -489,6 +489,10 @@ cache: attributes: # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' if you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" + ts_latest: + # Will enable cache-aside strategy for SQL timeseries latest DAO. + # make sure that if cache.type is 'redis' and cache.ts_latest.enabled is 'true' if you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' + enabled: "${CACHE_TS_LATEST_ENABLED:true}" specs: relations: timeToLiveInMinutes: "${CACHE_SPECS_RELATIONS_TTL:1440}" # Relations cache TTL @@ -539,6 +543,9 @@ cache: attributes: timeToLiveInMinutes: "${CACHE_SPECS_ATTRIBUTES_TTL:1440}" # Attributes cache TTL maxSize: "${CACHE_SPECS_ATTRIBUTES_MAX_SIZE:100000}" # 0 means the cache is disabled + tsLatest: + timeToLiveInMinutes: "${CACHE_SPECS_TS_LATEST_TTL:1440}" # Timeseries latest cache TTL + maxSize: "${CACHE_SPECS_TS_LATEST_MAX_SIZE:100000}" # 0 means the cache is disabled userSessionsInvalidation: # The value of this TTL is ignored and replaced by the JWT refresh token expiration time timeToLiveInMinutes: "0" diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..2c6a7d23fa --- /dev/null +++ b/build.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e # exit on any error + +#PROJECTS="msa/tb-node,msa/web-ui,rule-engine-pe/rule-node-twilio-sms" +PROJECTS="" + +if [ "$1" ]; then + PROJECTS="--projects $1" +fi + +echo "Building and pushing [amd64,arm64] projects '$PROJECTS' ..." +echo "HELP: usage ./build.sh [projects]" +echo "HELP: example ./build.sh msa/web-ui,msa/web-report" +java -version +#echo "Cleaning ui-ngx/node_modules" && rm -rf ui-ngx/node_modules + +MAVEN_OPTS="-Xmx1024m" NODE_OPTIONS="--max_old_space_size=4096" DOCKER_CLI_EXPERIMENTAL=enabled DOCKER_BUILDKIT=0 \ +mvn -T2 license:format clean install -DskipTests \ + $PROJECTS --also-make +# \ +# -Dpush-docker-amd-arm-images +# -Ddockerfile.skip=false -Dpush-docker-image=true +# --offline +# --projects '!msa/web-report' --also-make + +# push all +# mvn -T 1C license:format clean install -DskipTests -Ddockerfile.skip=false -Dpush-docker-image=true + + +## Build and push AMD and ARM docker images using docker buildx +## Reference to article how to setup docker miltiplatform build environment: https://medium.com/@artur.klauser/building-multi-architecture-docker-images-with-buildx-27d80f7e2408 +## install docker-ce from docker repo https://docs.docker.com/engine/install/ubuntu/ +# sudo apt install -y qemu-user-static binfmt-support +# export DOCKER_CLI_EXPERIMENTAL=enabled +# docker version +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# docker buildx create --name mybuilder +# docker buildx use mybuilder +# docker buildx inspect --bootstrap +# docker buildx ls +# mvn clean install -P push-docker-amd-arm-images \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbJavaRedisSerializer.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbJavaRedisSerializer.java new file mode 100644 index 0000000000..92c9f37900 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbJavaRedisSerializer.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; + +public class TbJavaRedisSerializer implements TbRedisSerializer { + + final RedisSerializer serializer = RedisSerializer.java(); + + @Override + public byte[] serialize(V value) throws SerializationException { + return serializer.serialize(value); + } + + @Override + public V deserialize(K key, byte[] bytes) throws SerializationException { + return (V) serializer.deserialize(bytes); + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDaoCachedRedis.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDaoCachedRedis.java new file mode 100644 index 0000000000..634302a1f3 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDaoCachedRedis.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("('${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale') && '${cache.ts_latest.enabled:false}'=='true' && '${cache.type:caffeine}'=='redis' ") +public @interface SqlTsLatestAnyDaoCachedRedis { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 9fda1c6bb5..50e02fca0a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -34,6 +34,7 @@ public class CacheConstants { public static final String ASSET_PROFILE_CACHE = "assetProfiles"; public static final String ATTRIBUTES_CACHE = "attributes"; + public static final String TS_LATEST_CACHE = "tsLatest"; public static final String USERS_SESSION_INVALIDATION_CACHE = "userSessionsInvalidation"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java new file mode 100644 index 0000000000..6b32808f7d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.ListenableFuture; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.stats.DefaultCounter; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.cache.CacheExecutorService; +import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; +import org.thingsboard.server.dao.timeseries.TsLatestCacheKey; +import org.thingsboard.server.dao.util.SqlTsLatestAnyDaoCachedRedis; + +import java.util.List; +import java.util.Optional; + +@Slf4j +@Component +@SqlTsLatestAnyDaoCachedRedis +@RequiredArgsConstructor +@Primary +public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao implements TimeseriesLatestDao { + public static final String STATS_NAME = "ts_latest.cache"; + + DefaultCounter hitCounter; + DefaultCounter missCounter; + + final CacheExecutorService cacheExecutorService; + final SqlTimeseriesLatestDao sqlDao; + final StatsFactory statsFactory; + final TbTransactionalCache cache; + + @PostConstruct + public void init() { + log.info("Init Redis cache-aside SQL Timeseries Latest DAO"); + this.hitCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "hit"); + this.missCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "miss"); + } + + @Override + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + return sqlDao.saveLatest(tenantId, entityId, tsKvEntry); + } + + @Override + public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return sqlDao.removeLatest(tenantId, entityId, query); + } + + @Override + public ListenableFuture> findLatestOpt(TenantId tenantId, EntityId entityId, String key) { + return sqlDao.findLatestOpt(tenantId, entityId, key); + } + + @Override + public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { + return sqlDao.findLatest(tenantId, entityId, key); + } + + @Override + public TsKvEntry findLatestSync(TenantId tenantId, EntityId entityId, String key) { + return sqlDao.findLatestSync(tenantId, entityId, key); + } + + @Override + public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { + return sqlDao.findAllLatest(tenantId, entityId); + } + + @Override + public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { + return sqlDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId); + } + + @Override + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { + return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestCacheKey.java new file mode 100644 index 0000000000..adb572922a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestCacheKey.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.EntityId; + +import java.io.Serial; +import java.io.Serializable; + +@EqualsAndHashCode +@Getter +@AllArgsConstructor +public class TsLatestCacheKey implements Serializable { + private static final long serialVersionUID = 2024369077925351881L; + + private final EntityId entityId; + private final String key; + + @Override + public String toString() { + return "{" + entityId + "}" + key; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java new file mode 100644 index 0000000000..d67e9272e3 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbJavaRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("TsLatestCache") +public class TsLatestRedisCache extends RedisTbTransactionalCache { + + public TsLatestRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJavaRedisSerializer<>()); } + +} diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 73c1495af8..c34d0ad592 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -10,6 +10,7 @@ audit-log.sink.type=none #cache.type=caffeine # will be injected redis by RedisContainer or will be default (caffeine) cache.maximumPoolSize=16 cache.attributes.enabled=true +cache.ts_latest.enabled=true cache.specs.relations.timeToLiveInMinutes=1440 cache.specs.relations.maxSize=100000 @@ -53,6 +54,9 @@ cache.specs.assetProfiles.maxSize=100000 cache.specs.attributes.timeToLiveInMinutes=1440 cache.specs.attributes.maxSize=100000 +cache.specs.tsLatest.timeToLiveInMinutes=1440 +cache.specs.tsLatest.maxSize=100000 + cache.specs.tokensOutdatageTime.timeToLiveInMinutes=1440 cache.specs.tokensOutdatageTime.maxSize=100000 diff --git a/dao/src/test/resources/logback.xml b/dao/src/test/resources/logback.xml index 5e293b2982..8d4951658e 100644 --- a/dao/src/test/resources/logback.xml +++ b/dao/src/test/resources/logback.xml @@ -9,6 +9,7 @@ + From 44096a7cb7059c1f3d847d5af736dbe4c4c4a7d5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 8 Apr 2024 14:57:20 +0200 Subject: [PATCH 04/42] TS latest dao put to Redis WIP --- .../cache/RedisTbTransactionalCache.java | 6 ++++- .../CachedRedisSqlTimeseriesLatestDao.java | 22 ++++++++++++++++++- .../dao/timeseries/TsLatestRedisCache.java | 19 ++++++++++++++-- .../{logback.xml => logback-test.xml} | 2 ++ 4 files changed, 45 insertions(+), 4 deletions(-) rename dao/src/test/resources/{logback.xml => logback-test.xml} (87%) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index cd5db13995..bb5af26c3e 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -214,8 +214,12 @@ public abstract class RedisTbTransactionalCache saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - return sqlDao.saveLatest(tenantId, entityId, tsKvEntry); + ListenableFuture future = sqlDao.saveLatest(tenantId, entityId, tsKvEntry); + future = Futures.transform(future, x -> { + cache.put(new TsLatestCacheKey(entityId, tsKvEntry.getKey()), tsKvEntry); + return x; + }, + cacheExecutorService); + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Void result) { + log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry); + } + + @Override + public void onFailure(Throwable t) { + log.trace("saveLatest onFailure [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry, t); + } + }, MoreExecutors.directExecutor()); + return future; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java index d67e9272e3..c0bca607c9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java @@ -15,8 +15,10 @@ */ package org.thingsboard.server.dao.timeseries; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.RedisStringCommands; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; @@ -25,11 +27,24 @@ import org.thingsboard.server.cache.TbJavaRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.TsKvEntry; +import java.io.Serializable; + @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("TsLatestCache") -public class TsLatestRedisCache extends RedisTbTransactionalCache { +@Slf4j +public class TsLatestRedisCache extends RedisTbTransactionalCache { public TsLatestRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJavaRedisSerializer<>()); } + super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJavaRedisSerializer<>()); + } + + @Override + public void put(TsLatestCacheKey key, TsKvEntry value) { + log.trace("put [{}][{}]", key, value); + final byte[] rawKey = getRawKey(key); + try (var connection = getConnection(rawKey)) { + put(connection, rawKey, value, RedisStringCommands.SetOption.UPSERT); + } + } } diff --git a/dao/src/test/resources/logback.xml b/dao/src/test/resources/logback-test.xml similarity index 87% rename from dao/src/test/resources/logback.xml rename to dao/src/test/resources/logback-test.xml index 8d4951658e..d7cff4b67b 100644 --- a/dao/src/test/resources/logback.xml +++ b/dao/src/test/resources/logback-test.xml @@ -10,6 +10,8 @@ + + From 3003fccf3b12f4f390b6e7860acbe1b5fc43db3b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 16 Apr 2024 16:43:20 +0200 Subject: [PATCH 05/42] BaseTimeseriesServiceTest refactored --- .../timeseries/BaseTimeseriesServiceTest.java | 31 +++---------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 390cb9afac..58b28872be 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.service.timeseries; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.extern.slf4j.Slf4j; +import org.assertj.core.data.Offset; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -32,6 +33,7 @@ import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; @@ -50,15 +52,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * @author Andrew Shvayka @@ -89,6 +90,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { KvEntry booleanKvEntry = new BooleanDataEntry(BOOLEAN_KEY, Boolean.TRUE); protected TenantId tenantId; + DeviceId deviceId = new DeviceId(Uuids.timeBased()); @Before public void before() { @@ -106,8 +108,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindAllLatest() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 1); saveEntries(deviceId, TS); @@ -150,8 +150,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindLatest() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 1); saveEntries(deviceId, TS); @@ -163,8 +161,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindLatestWithoutLatestUpdate() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 1); saveEntriesWithoutLatest(deviceId, TS); @@ -176,8 +172,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQueryAscOrder() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS - 3); saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 1); @@ -202,7 +196,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); saveEntries(deviceId, TS); saveEntries(deviceId, TS + 1L); @@ -222,7 +215,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); for (long i = TS; i <= TS + 100L; i += 10L) { saveEntries(deviceId, i); @@ -244,7 +236,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); for (long i = TS; i <= TS + 100000L; i += 10000L) { saveEntries(deviceId, i); @@ -268,7 +259,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS - 1L); for (long i = TS; i <= TS + 80000L; i += 10000L) { saveEntries(deviceId, i); @@ -292,7 +282,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) { saveEntries(deviceId, i); } @@ -314,7 +303,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) { saveEntries(deviceId, i); } @@ -336,8 +324,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindByQueryDescOrder() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, TS - 3); saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 1); @@ -362,7 +348,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindAllByQueries_verifyQueryId() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS); saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 10); @@ -373,7 +358,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindAllByQueries_verifyQueryId_forEntityView() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); saveEntries(deviceId, TS); saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 12); @@ -392,8 +376,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testDeleteDeviceTsDataWithOverwritingLatest() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); - saveEntries(deviceId, 10000); saveEntries(deviceId, 20000); saveEntries(deviceId, 30000); @@ -412,7 +394,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindDeviceTsData() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); List entries = new ArrayList<>(); entries.add(save(deviceId, 5000, 100)); @@ -563,7 +544,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindDeviceLongAndDoubleTsData() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); List entries = new ArrayList<>(); entries.add(save(deviceId, 5000, 100)); @@ -654,8 +634,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testSaveTs_RemoveTs_AndSaveTsAgain() throws Exception { - DeviceId deviceId = new DeviceId(Uuids.timeBased()); - save(deviceId, 2000000L, 95); save(deviceId, 4000000L, 100); save(deviceId, 6000000L, 105); @@ -686,7 +664,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { BasicTsKvEntry jsonEntry = new BasicTsKvEntry(TimeUnit.MINUTES.toMillis(5), new JsonDataEntry("test", "{\"test\":\"testValue\"}")); List timeseries = List.of(booleanEntry, stringEntry, longEntry, doubleEntry, jsonEntry); - DeviceId deviceId = new DeviceId(Uuids.timeBased()); for (TsKvEntry tsKvEntry : timeseries) { save(tenantId, deviceId, tsKvEntry); } From 771d15a6ddc71d1e95511c17c3b03ad006e750c3 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 16 Apr 2024 16:43:47 +0200 Subject: [PATCH 06/42] BaseTimeseriesServiceTest: 3 findLatest related tests added --- .../timeseries/BaseTimeseriesServiceTest.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 58b28872be..83db90ad71 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -159,6 +159,36 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(0)); } + @Test + public void testFindLatestOpt() throws Exception { + saveEntries(deviceId, TS - 2); + saveEntries(deviceId, TS - 1); + saveEntries(deviceId, TS); + + Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); + assertThat(entryOpt).isNotNull().isPresent(); + Assert.assertEquals(toTsEntry(TS, stringKvEntry), entryOpt.get()); + } + + @Test + public void testFindLatest_NotFound() throws Exception { + List entries = tsService.findLatest(tenantId, deviceId, Collections.singleton(STRING_KEY)).get(MAX_TIMEOUT, TimeUnit.SECONDS); + assertThat(entries).hasSize(1); + TsKvEntry tsKvEntry = entries.get(0); + assertThat(tsKvEntry).isNotNull(); + // null ts latest representation + assertThat(tsKvEntry.getKey()).isEqualTo(STRING_KEY); + assertThat(tsKvEntry.getDataType()).isEqualTo(DataType.STRING); + assertThat(tsKvEntry.getValue()).isNull(); + assertThat(tsKvEntry.getTs()).isCloseTo(System.currentTimeMillis(), Offset.offset(TimeUnit.MINUTES.toMillis(1))); + } + + @Test + public void testFindLatestOpt_NotFound() throws Exception { + Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); + assertThat(entryOpt).isNotNull().isNotPresent(); + } + @Test public void testFindLatestWithoutLatestUpdate() throws Exception { saveEntries(deviceId, TS - 2); From 034b480a6ca641f9b2670c146b0a78e8190c72be Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 16 Apr 2024 16:47:27 +0200 Subject: [PATCH 07/42] RedisTbTransactionalCache refactored to make doGet using the rawKey and the Redis connection precalculated slot based on hash --- .../thingsboard/server/cache/RedisTbTransactionalCache.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index bb5af26c3e..35ef07f0ce 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -78,7 +78,7 @@ public abstract class RedisTbTransactionalCache get(K key) { try (var connection = connectionFactory.getConnection()) { byte[] rawKey = getRawKey(key); - byte[] rawValue = connection.stringCommands().get(rawKey); + byte[] rawValue = doGet(connection, rawKey); if (rawValue == null) { return null; } else if (Arrays.equals(rawValue, BINARY_NULL_VALUE)) { @@ -95,6 +95,10 @@ public abstract class RedisTbTransactionalCache Date: Tue, 16 Apr 2024 16:50:36 +0200 Subject: [PATCH 08/42] SqlTimeseriesLatestDao refactored to use a common doFindLatestSync for any find request --- .../server/dao/sqlts/SqlTimeseriesLatestDao.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index a9fb85560a..0f773e83d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -155,17 +155,18 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme @Override public ListenableFuture> findLatestOpt(TenantId tenantId, EntityId entityId, String key) { - return service.submit(() -> Optional.ofNullable(doFindLatest(entityId, key))); + return service.submit(() -> Optional.ofNullable(doFindLatestSync(entityId, key))); } @Override public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - return service.submit(() -> getLatestTsKvEntry(entityId, key)); + log.trace("findLatest [{}][{}][{}]", tenantId, entityId, key); + return service.submit(() -> wrapNullTsKvEntry(key, doFindLatestSync(entityId, key))); } @Override public TsKvEntry findLatestSync(TenantId tenantId, EntityId entityId, String key) { - return getLatestTsKvEntry(entityId, key); + return wrapNullTsKvEntry(key, doFindLatestSync(entityId, key)); } @Override @@ -209,7 +210,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme ReadTsKvQueryResult::getData, MoreExecutors.directExecutor()); } - protected TsKvEntry doFindLatest(EntityId entityId, String key) { + protected TsKvEntry doFindLatestSync(EntityId entityId, String key) { TsKvLatestCompositeKey compositeKey = new TsKvLatestCompositeKey( entityId.getId(), @@ -225,7 +226,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme } protected ListenableFuture getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture latestFuture = service.submit(() -> doFindLatest(entityId, query.getKey())); + ListenableFuture latestFuture = service.submit(() -> doFindLatestSync(entityId, query.getKey())); return Futures.transformAsync(latestFuture, latest -> { if (latest == null) { return Futures.immediateFuture(new TsKvLatestRemovingResult(query.getKey(), false)); @@ -266,10 +267,9 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return tsLatestQueue.add(latestEntity); } - private TsKvEntry getLatestTsKvEntry(EntityId entityId, String key) { - TsKvEntry latest = doFindLatest(entityId, key); + protected TsKvEntry wrapNullTsKvEntry(final String key, final TsKvEntry latest) { if (latest == null) { - latest = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); + return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); } return latest; } From 551325b8c7afe8afa58ed2586f6d3e6258b0894e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 16 Apr 2024 16:52:31 +0200 Subject: [PATCH 09/42] added CachedRedisSqlTimeseriesLatestDao and TsLatestRedisCache (prototype impl with string commands). Caffeine cache not implemented yet --- .../CachedRedisSqlTimeseriesLatestDao.java | 99 +++++++++++++++---- .../dao/timeseries/TsLatestRedisCache.java | 53 ++++++++++ 2 files changed, 134 insertions(+), 18 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index b283832186..482b456e60 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -24,6 +24,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; +import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -48,14 +49,12 @@ import java.util.Optional; @Primary public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao implements TimeseriesLatestDao { public static final String STATS_NAME = "ts_latest.cache"; - - DefaultCounter hitCounter; - DefaultCounter missCounter; - final CacheExecutorService cacheExecutorService; final SqlTimeseriesLatestDao sqlDao; final StatsFactory statsFactory; final TbTransactionalCache cache; + DefaultCounter hitCounter; + DefaultCounter missCounter; @PostConstruct public void init() { @@ -72,37 +71,101 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return x; }, cacheExecutorService); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(Void result) { - log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry); - } - - @Override - public void onFailure(Throwable t) { - log.trace("saveLatest onFailure [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry, t); - } - }, MoreExecutors.directExecutor()); + if (log.isTraceEnabled()) { + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(Void result) { + log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry); + } + + @Override + public void onFailure(Throwable t) { + log.info("saveLatest onFailure [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry, t); + } + }, MoreExecutors.directExecutor()); + } return future; } @Override public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return sqlDao.removeLatest(tenantId, entityId, query); + ListenableFuture future = sqlDao.removeLatest(tenantId, entityId, query); + future = Futures.transform(future, x -> { + cache.evict(new TsLatestCacheKey(entityId, query.getKey())); + return x; + }, + cacheExecutorService); + if (log.isTraceEnabled()) { + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(TsKvLatestRemovingResult result) { + log.trace("removeLatest onSuccess [{}][{}][{}]", entityId, query.getKey(), query); + } + + @Override + public void onFailure(Throwable t) { + log.info("removeLatest onFailure [{}][{}][{}]", entityId, query.getKey(), query, t); + } + }, MoreExecutors.directExecutor()); + } + return future; } @Override public ListenableFuture> findLatestOpt(TenantId tenantId, EntityId entityId, String key) { - return sqlDao.findLatestOpt(tenantId, entityId, key); + log.trace("findLatestOpt"); + return doFindLatest(tenantId, entityId, key); } @Override public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - return sqlDao.findLatest(tenantId, entityId, key); + return Futures.transform(doFindLatest(tenantId, entityId, key), x -> sqlDao.wrapNullTsKvEntry(key, x.orElse(null)), MoreExecutors.directExecutor()); + } + + public ListenableFuture> doFindLatest(TenantId tenantId, EntityId entityId, String key) { + final TsLatestCacheKey cacheKey = new TsLatestCacheKey(entityId, key); + ListenableFuture> cacheFuture = cacheExecutorService.submit(() -> cache.get(cacheKey)); + + return Futures.transformAsync(cacheFuture, (cacheValueWrap) -> { + if (cacheValueWrap != null) { + final TsKvEntry tsKvEntry = cacheValueWrap.get(); + log.debug("findLatest cache hit [{}][{}][{}]", entityId, key, tsKvEntry); + return Futures.immediateFuture(Optional.ofNullable(tsKvEntry)); + } + log.debug("findLatest cache miss [{}][{}]", entityId, key); + ListenableFuture> daoFuture = sqlDao.findLatestOpt(tenantId,entityId, key); + + return Futures.transformAsync(daoFuture, (daoValue) -> { + + if (daoValue.isEmpty()) { + //TODO implement the cache logic if no latest found in TS DAO. Currently we are always getting from DB to stay on the safe side + return Futures.immediateFuture(daoValue); + } + ListenableFuture> cachePutFuture = cacheExecutorService.submit(() -> { + cache.put(new TsLatestCacheKey(entityId, key), daoValue.get()); + return daoValue; + }); + + Futures.addCallback(cachePutFuture, new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, key, result); + } + + @Override + public void onFailure(Throwable t) { + log.info("saveLatest onFailure [{}][{}][{}]", entityId, key, daoValue, t); + } + + }, MoreExecutors.directExecutor()); + return cachePutFuture; + }, MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); } @Override public TsKvEntry findLatestSync(TenantId tenantId, EntityId entityId, String key) { + log.trace("findLatestSync DEPRECATED [{}][{}]", entityId, key); return sqlDao.findLatestSync(tenantId, entityId, key); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java index c0bca607c9..61d2c8bdb2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java @@ -16,18 +16,24 @@ package org.thingsboard.server.dao.timeseries; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.NotImplementedException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.connection.RedisStringCommands; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbCacheTransaction; +import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.TbJavaRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.io.Serializable; +import java.util.Collection; +import java.util.List; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("TsLatestCache") @@ -47,4 +53,51 @@ public class TsLatestRedisCache } } + @Override + public void putIfAbsent(TsLatestCacheKey key, TsKvEntry value) { + log.trace("putIfAbsent [{}][{}]", key, value); + throw new NotImplementedException("putIfAbsent is not supported by TsLatestRedisCache"); + } + + @Override + public TbCacheValueWrapper get(TsLatestCacheKey key) { + log.debug("get [{}]", key); + return super.get(key); + } + + @Override + protected byte[] doGet(RedisConnection connection, byte[] rawKey) { + log.trace("doGet [{}][{}]", connection, rawKey); + return connection.stringCommands().get(rawKey); + } + + @Override + public void evict(TsLatestCacheKey key){ + log.trace("evict [{}]", key); + final byte[] rawKey = getRawKey(key); + try (var connection = getConnection(rawKey)) { + connection.keyCommands().del(rawKey); + } + } + + @Override + public void evict(Collection keys) { + throw new NotImplementedException("evict by many keys is not supported by TsLatestRedisCache"); + } + + @Override + public void evictOrPut(TsLatestCacheKey key, TsKvEntry value) { + throw new NotImplementedException("evictOrPut is not supported by TsLatestRedisCache"); + } + + @Override + public TbCacheTransaction newTransactionForKey(TsLatestCacheKey key) { + throw new NotImplementedException("newTransactionForKey is not supported by TsLatestRedisCache"); + } + + @Override + public TbCacheTransaction newTransactionForKeys(List keys) { + throw new NotImplementedException("newTransactionForKeys is not supported by TsLatestRedisCache"); + } + } From c49d97f7d19d69883a789b2ac2b687f6c34f322a Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 16 Apr 2024 17:35:04 +0200 Subject: [PATCH 10/42] BaseTimeseriesServiceTest added overwrite TS cases and historical data load case, that fails without condition on update only if ts is the latest --- .../timeseries/BaseTimeseriesServiceTest.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 83db90ad71..d32933684a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -159,6 +159,40 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(0)); } + @Test + public void testFindLatestOpt_givenSaveWithHistoricalNonOrderedTS() throws Exception { + save(tenantId, deviceId, toTsEntry(TS - 1, stringKvEntry)); + save(tenantId, deviceId, toTsEntry(TS, stringKvEntry)); + save(tenantId, deviceId, toTsEntry(TS - 10, stringKvEntry)); + save(tenantId, deviceId, toTsEntry(TS - 11, stringKvEntry)); + + Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); + assertThat(entryOpt).isNotNull().isPresent(); + Assert.assertEquals(toTsEntry(TS, stringKvEntry), entryOpt.orElse(null)); + } + + @Test + public void testFindLatestOpt_givenSaveWithSameTSOverwriteValue() throws Exception { + save(tenantId, deviceId, toTsEntry(TS, new StringDataEntry(STRING_KEY, "old"))); + save(tenantId, deviceId, toTsEntry(TS, new StringDataEntry(STRING_KEY, "new"))); + + Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); + assertThat(entryOpt).isNotNull().isPresent(); + Assert.assertEquals(toTsEntry(TS, new StringDataEntry(STRING_KEY, "new")), entryOpt.orElse(null)); + } + + public void testFindLatestOpt_givenSaveWithSameTSOverwriteTypeAndValue() throws Exception { + save(tenantId, deviceId, toTsEntry(TS, new JsonDataEntry("temp", "{\"hello\":\"world\"}"))); + save(tenantId, deviceId, toTsEntry(TS, new BooleanDataEntry("temp", true))); + save(tenantId, deviceId, toTsEntry(TS, new LongDataEntry("temp", 100L))); + save(tenantId, deviceId, toTsEntry(TS, new DoubleDataEntry("temp", Math.PI))); + save(tenantId, deviceId, toTsEntry(TS, new StringDataEntry("temp", "NOOP"))); + + Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); + assertThat(entryOpt).isNotNull().isPresent(); + Assert.assertEquals(toTsEntry(TS, new StringDataEntry("temp", "NOOP")), entryOpt.orElse(null)); + } + @Test public void testFindLatestOpt() throws Exception { saveEntries(deviceId, TS - 2); From a4469f69539ead14b1c926ce103cfdb704741823 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 17 Apr 2024 12:14:42 +0200 Subject: [PATCH 11/42] TsLatestRedisCache: added LUA upsert script. load script, eval, evalsha, test for script sha, fetch latest by zRange Redis command --- .../cache/RedisTbTransactionalCache.java | 1 + .../dao/timeseries/TsLatestRedisCache.java | 75 +++++++++++++++---- .../timeseries/TsLatestRedisCacheTest.java | 30 ++++++++ 3 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index 35ef07f0ce..4277a7bab9 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -51,6 +51,7 @@ public abstract class RedisTbTransactionalCache keySerializer = StringRedisSerializer.UTF_8; private final TbRedisSerializer valueSerializer; diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java index 61d2c8bdb2..e9f9ab522c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java @@ -15,12 +15,15 @@ */ package org.thingsboard.server.dao.timeseries; +import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.NotImplementedException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.connection.RedisStringCommands; +import org.springframework.data.redis.connection.ReturnType; +import org.springframework.data.redis.serializer.StringRedisSerializer; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; @@ -32,33 +35,42 @@ import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.io.Serializable; +import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Set; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("TsLatestCache") @Slf4j public class TsLatestRedisCache extends RedisTbTransactionalCache { + static final byte[] UPSERT_TS_LATEST_LUA_SCRIPT = StringRedisSerializer.UTF_8.serialize("" + + "redis.call('ZREMRANGEBYSCORE', KEYS[1], ARGV[1], ARGV[1]); " + + "redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2]); " + + "local current_size = redis.call('ZCARD', KEYS[1]); " + + "if current_size > 1 then" + + " redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -2) " + + "end;"); + static final byte[] UPSERT_TS_LATEST_SHA = StringRedisSerializer.UTF_8.serialize("24e226c3ea34e3e850113e8eb1f3cd2b88171988"); + public TsLatestRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJavaRedisSerializer<>()); } - @Override - public void put(TsLatestCacheKey key, TsKvEntry value) { - log.trace("put [{}][{}]", key, value); - final byte[] rawKey = getRawKey(key); - try (var connection = getConnection(rawKey)) { - put(connection, rawKey, value, RedisStringCommands.SetOption.UPSERT); + @PostConstruct + public void init() { + try (var connection = getConnection(UPSERT_TS_LATEST_SHA)) { + log.debug("Loading LUA with expected SHA[{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), connection.getNativeConnection()); + String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT); + if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { + log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), sha, connection.getNativeConnection()); + } + } catch (Throwable t) { + log.error("Error on Redis TS Latest cache init", t); } } - @Override - public void putIfAbsent(TsLatestCacheKey key, TsKvEntry value) { - log.trace("putIfAbsent [{}][{}]", key, value); - throw new NotImplementedException("putIfAbsent is not supported by TsLatestRedisCache"); - } - @Override public TbCacheValueWrapper get(TsLatestCacheKey key) { log.debug("get [{}]", key); @@ -68,11 +80,38 @@ public class TsLatestRedisCache @Override protected byte[] doGet(RedisConnection connection, byte[] rawKey) { log.trace("doGet [{}][{}]", connection, rawKey); - return connection.stringCommands().get(rawKey); + Set values = connection.commands().zRange(rawKey, -1, -1); + return values == null ? null : values.stream().findFirst().orElse(null); } @Override - public void evict(TsLatestCacheKey key){ + public void put(TsLatestCacheKey key, TsKvEntry value) { + log.trace("put [{}][{}]", key, value); + final byte[] rawKey = getRawKey(key); + try (var connection = getConnection(rawKey)) { + byte[] rawValue = getRawValue(value); + byte[] ts = StringRedisSerializer.UTF_8.serialize(String.valueOf(value.toTsValue().getTs())); + try { + connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue); + } catch (InvalidDataAccessApiUsageException e) { + log.debug("loading LUA [{}]", connection.getNativeConnection()); + String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT); + if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { + log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}]", new String(UPSERT_TS_LATEST_SHA), sha); + } + try { + connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue); + } catch (InvalidDataAccessApiUsageException ignored) { + log.debug("Slowly executing eval instead of fast evalsha"); + connection.scriptingCommands().eval(UPSERT_TS_LATEST_LUA_SCRIPT, ReturnType.VALUE, 1, rawKey, ts, rawValue); + } + + } + } + } + + @Override + public void evict(TsLatestCacheKey key) { log.trace("evict [{}]", key); final byte[] rawKey = getRawKey(key); try (var connection = getConnection(rawKey)) { @@ -80,6 +119,12 @@ public class TsLatestRedisCache } } + @Override + public void putIfAbsent(TsLatestCacheKey key, TsKvEntry value) { + log.trace("putIfAbsent [{}][{}]", key, value); + throw new NotImplementedException("putIfAbsent is not supported by TsLatestRedisCache"); + } + @Override public void evict(Collection keys) { throw new NotImplementedException("evict by many keys is not supported by TsLatestRedisCache"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java new file mode 100644 index 0000000000..5de26c15d3 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java @@ -0,0 +1,30 @@ +package org.thingsboard.server.dao.timeseries; + +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; + +import java.security.MessageDigest; + +import static org.assertj.core.api.Assertions.assertThat; + +class TsLatestRedisCacheTest { + + @Test + void testUpsertTsLatestLUAScriptHash() { + assertThat(getSHA1(TsLatestRedisCache.UPSERT_TS_LATEST_LUA_SCRIPT)).isEqualTo(new String(TsLatestRedisCache.UPSERT_TS_LATEST_SHA)); + } + + @SneakyThrows + String getSHA1(byte[] script) { + MessageDigest md = MessageDigest.getInstance("SHA-1"); + byte[] hash = md.digest(script); + + StringBuilder sb = new StringBuilder(); + for (byte b : hash) { + sb.append(String.format("%02x", b)); + } + + return sb.toString(); + } + +} \ No newline at end of file From 67c2392cccea99861bb1f869d4dce55f58a81f3a Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 17 Apr 2024 12:15:22 +0200 Subject: [PATCH 12/42] RedisClusterSqlTestSuite introduced --- .../dao/AbstractRedisClusterContainer.java | 92 +++++++++++++++++++ .../server/dao/RedisClusterSqlTestSuite.java | 31 +++++++ 2 files changed, 123 insertions(+) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/RedisClusterSqlTestSuite.java diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java new file mode 100644 index 0000000000..cfa4c00ddf --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import lombok.extern.slf4j.Slf4j; +import org.junit.ClassRule; +import org.junit.rules.ExternalResource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.OutputFrame; +import redis.clients.jedis.Jedis; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class AbstractRedisClusterContainer { + + static final String nodes = "127.0.0.1:6371,127.0.0.1:6372,127.0.0.1:6373,127.0.0.1:6374,127.0.0.1:6375,127.0.0.1:6376"; + + @ClassRule(order = 0) + public static Network network = Network.newNetwork(); + @ClassRule(order = 1) + public static GenericContainer redis1 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6371").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + @ClassRule(order = 2) + public static GenericContainer redis2 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6372").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + @ClassRule(order = 3) + public static GenericContainer redis3 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6373").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + @ClassRule(order = 4) + public static GenericContainer redis4 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6374").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + @ClassRule(order = 5) + public static GenericContainer redis5 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6375").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + @ClassRule(order = 6) + public static GenericContainer redis6 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6376").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + + + @ClassRule(order = 100) + public static ExternalResource resource = new ExternalResource() { + @Override + protected void before() throws Throwable { + redis1.start(); + redis2.start(); + redis3.start(); + redis4.start(); + redis5.start(); + redis6.start(); + + String clusterCreateCommand = "echo yes | redis-cli --cluster create " + + "127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375 127.0.0.1:6376 " + + "--cluster-replicas 1"; + + log.warn("Command to init Redis Cluster: {}", clusterCreateCommand); + log.warn("Connect to nodes : {}", nodes); + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); + var result = redis6.execInContainer("/bin/sh", "-c", clusterCreateCommand); + log.warn("Init cluster result: {}", result); + + System.setProperty("cache.type", "redis"); + System.setProperty("redis.connection.type", "cluster"); + System.setProperty("redis.cluster.nodes", nodes); + System.setProperty("redis.password", ""); + + System.setProperty("redis.cluster.useDefaultPoolConfig", "false"); + } + + @Override + protected void after() { + redis1.stop(); + redis2.stop(); + redis3.stop(); + redis4.stop(); + redis5.stop(); + redis6.stop(); + List.of("cache.type", "redis.connection.type", "redis.standalone.host", "redis.standalone.port") + .forEach(System.getProperties()::remove); + } + }; + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/RedisClusterSqlTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/RedisClusterSqlTestSuite.java new file mode 100644 index 0000000000..be0bcc6fc7 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/RedisClusterSqlTestSuite.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; +import org.junit.runner.RunWith; + +@RunWith(ClasspathSuite.class) +@ClassnameFilters( + //All the same tests using redis instead of caffeine. + { + "org.thingsboard.server.dao.service.*ServiceSqlTest", + } +) +public class RedisClusterSqlTestSuite extends AbstractRedisClusterContainer { + +} From 743b8b1d801d751fd9fc55e31f5014853a670c5a Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 17 Apr 2024 12:16:31 +0200 Subject: [PATCH 13/42] logback-test temporary added TRACE for SqlTimeseriesLatestDao and CachedRedisSqlTimeseriesLatestDao --- dao/src/test/resources/logback-test.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dao/src/test/resources/logback-test.xml b/dao/src/test/resources/logback-test.xml index d7cff4b67b..87aa67a67e 100644 --- a/dao/src/test/resources/logback-test.xml +++ b/dao/src/test/resources/logback-test.xml @@ -10,6 +10,8 @@ + + From 168525d3b9e00636e979b36b75538ffb5ea41853 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 17 Apr 2024 12:24:16 +0200 Subject: [PATCH 14/42] mvn license:format --- .../dao/timeseries/TsLatestRedisCacheTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java index 5de26c15d3..c463b4630f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.timeseries; import lombok.SneakyThrows; From 92793360f304687cd48f8dbf5a95834efcda53a1 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 17 Apr 2024 12:39:54 +0200 Subject: [PATCH 15/42] CachedRedisSqlTimeseriesLatestDao.java removed deprecated findLatestSync --- .../server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index 482b456e60..32d0ee91c9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -163,12 +163,6 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries }, MoreExecutors.directExecutor()); } - @Override - public TsKvEntry findLatestSync(TenantId tenantId, EntityId entityId, String key) { - log.trace("findLatestSync DEPRECATED [{}][{}]", entityId, key); - return sqlDao.findLatestSync(tenantId, entityId, key); - } - @Override public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { return sqlDao.findAllLatest(tenantId, entityId); From 20ec210c320648a190d39ad40a78c08a6d4ce172 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 17 Apr 2024 12:43:14 +0200 Subject: [PATCH 16/42] AbstractRedisClusterContainer refactored --- .../server/dao/AbstractRedisClusterContainer.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java index cfa4c00ddf..a95f4b8c17 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java @@ -58,21 +58,19 @@ public class AbstractRedisClusterContainer { redis5.start(); redis6.start(); + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); // otherwise not all containers have time to start + String clusterCreateCommand = "echo yes | redis-cli --cluster create " + "127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375 127.0.0.1:6376 " + "--cluster-replicas 1"; - log.warn("Command to init Redis Cluster: {}", clusterCreateCommand); - log.warn("Connect to nodes : {}", nodes); - Thread.sleep(TimeUnit.SECONDS.toMillis(5)); var result = redis6.execInContainer("/bin/sh", "-c", clusterCreateCommand); log.warn("Init cluster result: {}", result); + log.warn("Connect to nodes: {}", nodes); System.setProperty("cache.type", "redis"); System.setProperty("redis.connection.type", "cluster"); System.setProperty("redis.cluster.nodes", nodes); - System.setProperty("redis.password", ""); - System.setProperty("redis.cluster.useDefaultPoolConfig", "false"); } @@ -84,7 +82,7 @@ public class AbstractRedisClusterContainer { redis4.stop(); redis5.stop(); redis6.stop(); - List.of("cache.type", "redis.connection.type", "redis.standalone.host", "redis.standalone.port") + List.of("cache.type", "redis.connection.type", "redis.cluster.nodes", "redis.cluster.useDefaultPoolConfig\"") .forEach(System.getProperties()::remove); } }; From c44ba6307c51cde84ea8f0253eb53f226ee6962d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 10 Jun 2024 16:02:32 +0200 Subject: [PATCH 17/42] added sequence numver for lates timeseries --- .../latest/InsertLatestTsRepository.java | 2 +- .../sql/SqlLatestInsertTsRepository.java | 285 +++++++++++------- .../main/resources/sql/schema-entities.sql | 3 + .../sql/LatestTimeseriesPerformanceTest.java | 151 ++++++++++ dao/src/test/resources/logback.xml | 1 + .../resources/sql/psql/drop-all-tables.sql | 1 + 6 files changed, 330 insertions(+), 113 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/InsertLatestTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/InsertLatestTsRepository.java index 0aa95fa324..d85b66a258 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/InsertLatestTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/InsertLatestTsRepository.java @@ -21,6 +21,6 @@ import java.util.List; public interface InsertLatestTsRepository { - void saveOrUpdate(List entities); + List saveOrUpdate(List entities); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index 530ac7ce23..48b54c2281 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -15,23 +15,28 @@ */ package org.thingsboard.server.dao.sqlts.insert.latest.sql; +import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.PreparedStatementCreator; +import org.springframework.jdbc.core.SqlProvider; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Repository; -import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; +import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; import java.util.ArrayList; import java.util.List; +import java.util.Map; @SqlTsLatestAnyDao @@ -44,129 +49,185 @@ public class SqlLatestInsertTsRepository extends AbstractInsertRepository implem private Boolean updateByLatestTs; private static final String BATCH_UPDATE = - "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json) WHERE entity_id = ? AND key = ?"; + "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), seq_number = nextval('ts_kv_latest_seq') WHERE entity_id = ? AND key = ?"; private static final String INSERT_OR_UPDATE = - "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " + - "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json)"; + "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, seq_number) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), nextval('ts_kv_latest_seq')) " + + "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), seq_number = nextval('ts_kv_latest_seq')"; private static final String BATCH_UPDATE_BY_LATEST_TS = BATCH_UPDATE + " AND ts_kv_latest.ts <= ?"; private static final String INSERT_OR_UPDATE_BY_LATEST_TS = INSERT_OR_UPDATE + " WHERE ts_kv_latest.ts <= ?"; + private static final String RETURNING = " RETURNING seq_number"; + + private static final String SEQ_NUMBER = "seq_number"; + + private String batchUpdateQuery; + private String insertOrUpdateQuery; + + @PostConstruct + private void init() { + this.batchUpdateQuery = (updateByLatestTs ? BATCH_UPDATE_BY_LATEST_TS : BATCH_UPDATE) + RETURNING; + this.insertOrUpdateQuery = (updateByLatestTs ? INSERT_OR_UPDATE_BY_LATEST_TS : INSERT_OR_UPDATE) + RETURNING; + } + @Override - public void saveOrUpdate(List entities) { - transactionTemplate.execute(new TransactionCallbackWithoutResult() { + public List saveOrUpdate(List entities) { + return transactionTemplate.execute(status -> { + List seqNumbers = new ArrayList<>(entities.size()); + + KeyHolder keyHolder = new GeneratedKeyHolder(); + + int[] updateResult = onBatchUpdate(entities, keyHolder); + + List> seqNumbersList = keyHolder.getKeyList(); + + int notUpdatedCount = entities.size() - seqNumbersList.size(); + + List toInsertIndexes = new ArrayList<>(notUpdatedCount); + List insertEntities = new ArrayList<>(notUpdatedCount); + int keyHolderIndex = 0; + for (int i = 0; i < updateResult.length; i++) { + if (updateResult[i] == 0) { + insertEntities.add(entities.get(i)); + seqNumbers.add(0L); + toInsertIndexes.add(i); + } else { + seqNumbers.add((Long) seqNumbersList.get(keyHolderIndex).get(SEQ_NUMBER)); + keyHolderIndex++; + } + } + + if (insertEntities.isEmpty()) { + return seqNumbers; + } + + onInsertOrUpdate(insertEntities, keyHolder); + + seqNumbersList = keyHolder.getKeyList(); + + for (int i = 0; i < seqNumbersList.size(); i++) { + seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(SEQ_NUMBER)); + } + + return seqNumbers; + }); + } + + private int[] onBatchUpdate(List entities, KeyHolder keyHolder) { + return jdbcTemplate.batchUpdate(new SimplePreparedStatementCreator(batchUpdateQuery), new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + TsKvLatestEntity tsKvLatestEntity = entities.get(i); + ps.setLong(1, tsKvLatestEntity.getTs()); + + if (tsKvLatestEntity.getBooleanValue() != null) { + ps.setBoolean(2, tsKvLatestEntity.getBooleanValue()); + } else { + ps.setNull(2, Types.BOOLEAN); + } + + ps.setString(3, replaceNullChars(tsKvLatestEntity.getStrValue())); + + if (tsKvLatestEntity.getLongValue() != null) { + ps.setLong(4, tsKvLatestEntity.getLongValue()); + } else { + ps.setNull(4, Types.BIGINT); + } + + if (tsKvLatestEntity.getDoubleValue() != null) { + ps.setDouble(5, tsKvLatestEntity.getDoubleValue()); + } else { + ps.setNull(5, Types.DOUBLE); + } + + ps.setString(6, replaceNullChars(tsKvLatestEntity.getJsonValue())); + + ps.setObject(7, tsKvLatestEntity.getEntityId()); + ps.setInt(8, tsKvLatestEntity.getKey()); + if (updateByLatestTs) { + ps.setLong(9, tsKvLatestEntity.getTs()); + } + } + @Override - protected void doInTransactionWithoutResult(TransactionStatus status) { - String batchUpdateQuery = updateByLatestTs ? BATCH_UPDATE_BY_LATEST_TS : BATCH_UPDATE; - String insertOrUpdateQuery = updateByLatestTs ? INSERT_OR_UPDATE_BY_LATEST_TS : INSERT_OR_UPDATE; - - int[] result = jdbcTemplate.batchUpdate(batchUpdateQuery, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - TsKvLatestEntity tsKvLatestEntity = entities.get(i); - ps.setLong(1, tsKvLatestEntity.getTs()); - - if (tsKvLatestEntity.getBooleanValue() != null) { - ps.setBoolean(2, tsKvLatestEntity.getBooleanValue()); - } else { - ps.setNull(2, Types.BOOLEAN); - } - - ps.setString(3, replaceNullChars(tsKvLatestEntity.getStrValue())); - - if (tsKvLatestEntity.getLongValue() != null) { - ps.setLong(4, tsKvLatestEntity.getLongValue()); - } else { - ps.setNull(4, Types.BIGINT); - } - - if (tsKvLatestEntity.getDoubleValue() != null) { - ps.setDouble(5, tsKvLatestEntity.getDoubleValue()); - } else { - ps.setNull(5, Types.DOUBLE); - } - - ps.setString(6, replaceNullChars(tsKvLatestEntity.getJsonValue())); - - ps.setObject(7, tsKvLatestEntity.getEntityId()); - ps.setInt(8, tsKvLatestEntity.getKey()); - if (updateByLatestTs) { - ps.setLong(9, tsKvLatestEntity.getTs()); - } - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); - - int updatedCount = 0; - for (int i = 0; i < result.length; i++) { - if (result[i] == 0) { - updatedCount++; - } + public int getBatchSize() { + return entities.size(); + } + }, keyHolder); + } + + private void onInsertOrUpdate(List insertEntities, KeyHolder keyHolder) { + jdbcTemplate.batchUpdate(new SimplePreparedStatementCreator(insertOrUpdateQuery), new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + TsKvLatestEntity tsKvLatestEntity = insertEntities.get(i); + ps.setObject(1, tsKvLatestEntity.getEntityId()); + ps.setInt(2, tsKvLatestEntity.getKey()); + + ps.setLong(3, tsKvLatestEntity.getTs()); + ps.setLong(9, tsKvLatestEntity.getTs()); + if (updateByLatestTs) { + ps.setLong(15, tsKvLatestEntity.getTs()); } - List insertEntities = new ArrayList<>(updatedCount); - for (int i = 0; i < result.length; i++) { - if (result[i] == 0) { - insertEntities.add(entities.get(i)); - } + if (tsKvLatestEntity.getBooleanValue() != null) { + ps.setBoolean(4, tsKvLatestEntity.getBooleanValue()); + ps.setBoolean(10, tsKvLatestEntity.getBooleanValue()); + } else { + ps.setNull(4, Types.BOOLEAN); + ps.setNull(10, Types.BOOLEAN); + } + + ps.setString(5, replaceNullChars(tsKvLatestEntity.getStrValue())); + ps.setString(11, replaceNullChars(tsKvLatestEntity.getStrValue())); + + if (tsKvLatestEntity.getLongValue() != null) { + ps.setLong(6, tsKvLatestEntity.getLongValue()); + ps.setLong(12, tsKvLatestEntity.getLongValue()); + } else { + ps.setNull(6, Types.BIGINT); + ps.setNull(12, Types.BIGINT); } - jdbcTemplate.batchUpdate(insertOrUpdateQuery, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - TsKvLatestEntity tsKvLatestEntity = insertEntities.get(i); - ps.setObject(1, tsKvLatestEntity.getEntityId()); - ps.setInt(2, tsKvLatestEntity.getKey()); - - ps.setLong(3, tsKvLatestEntity.getTs()); - ps.setLong(9, tsKvLatestEntity.getTs()); - if (updateByLatestTs) { - ps.setLong(15, tsKvLatestEntity.getTs()); - } - - if (tsKvLatestEntity.getBooleanValue() != null) { - ps.setBoolean(4, tsKvLatestEntity.getBooleanValue()); - ps.setBoolean(10, tsKvLatestEntity.getBooleanValue()); - } else { - ps.setNull(4, Types.BOOLEAN); - ps.setNull(10, Types.BOOLEAN); - } - - ps.setString(5, replaceNullChars(tsKvLatestEntity.getStrValue())); - ps.setString(11, replaceNullChars(tsKvLatestEntity.getStrValue())); - - if (tsKvLatestEntity.getLongValue() != null) { - ps.setLong(6, tsKvLatestEntity.getLongValue()); - ps.setLong(12, tsKvLatestEntity.getLongValue()); - } else { - ps.setNull(6, Types.BIGINT); - ps.setNull(12, Types.BIGINT); - } - - if (tsKvLatestEntity.getDoubleValue() != null) { - ps.setDouble(7, tsKvLatestEntity.getDoubleValue()); - ps.setDouble(13, tsKvLatestEntity.getDoubleValue()); - } else { - ps.setNull(7, Types.DOUBLE); - ps.setNull(13, Types.DOUBLE); - } - - ps.setString(8, replaceNullChars(tsKvLatestEntity.getJsonValue())); - ps.setString(14, replaceNullChars(tsKvLatestEntity.getJsonValue())); - } - - @Override - public int getBatchSize() { - return insertEntities.size(); - } - }); + if (tsKvLatestEntity.getDoubleValue() != null) { + ps.setDouble(7, tsKvLatestEntity.getDoubleValue()); + ps.setDouble(13, tsKvLatestEntity.getDoubleValue()); + } else { + ps.setNull(7, Types.DOUBLE); + ps.setNull(13, Types.DOUBLE); + } + + ps.setString(8, replaceNullChars(tsKvLatestEntity.getJsonValue())); + ps.setString(14, replaceNullChars(tsKvLatestEntity.getJsonValue())); } - }); + + @Override + public int getBatchSize() { + return insertEntities.size(); + } + }, keyHolder); + } + + private static class SimplePreparedStatementCreator implements PreparedStatementCreator, SqlProvider { + + private static final String[] COLUMNS = {SEQ_NUMBER}; + private final String sql; + + + public SimplePreparedStatementCreator(String sql) { + this.sql = sql; + } + + @Override + public PreparedStatement createPreparedStatement(Connection con) throws SQLException { + return con.prepareStatement(sql, COLUMNS); + } + + @Override + public String getSql() { + return this.sql; + } } } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index f2978b7f4a..b92c27574c 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -538,6 +538,8 @@ CREATE TABLE IF NOT EXISTS entity_view ( CONSTRAINT entity_view_external_id_unq_key UNIQUE (tenant_id, external_id) ); +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_seq cache 1000; + CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, @@ -548,6 +550,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, + seq_number bigint, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java new file mode 100644 index 0000000000..00f458ca52 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java @@ -0,0 +1,151 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.timeseries.sql; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.service.AbstractServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +@DaoSqlTest +public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { + + private static final String STRING_KEY = "stringKey"; + private static final String LONG_KEY = "longKey"; + private static final String DOUBLE_KEY = "doubleKey"; + private static final String BOOLEAN_KEY = "booleanKey"; + public static final int AMOUNT_OF_UNIQ_KEY = 10000; + + private final Random random = new Random(); + + @Autowired + private TimeseriesLatestDao timeseriesLatestDao; + + private ListeningExecutorService testExecutor; + + private EntityId entityId; + + private AtomicLong saveCounter; + + @Before + public void before() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + Assert.assertNotNull(savedTenant); + tenantId = savedTenant.getId(); + entityId = new DeviceId(UUID.randomUUID()); + saveCounter = new AtomicLong(0); + testExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(200, ThingsBoardThreadFactory.forName(getClass().getSimpleName() + "-test-scope"))); + } + + @After + public void after() { + tenantService.deleteTenant(tenantId); + if (testExecutor != null) { + testExecutor.shutdownNow(); + } + } + + @Test + public void test_save_latest_timeseries() throws Exception { + warmup(); + saveCounter.set(0); + + long startTime = System.currentTimeMillis(); + List> futures = new ArrayList<>(); + for (int i = 0; i < 25_000; i++) { + futures.add(save(generateStrEntry(getRandomKey()))); + futures.add(save(generateLngEntry(getRandomKey()))); + futures.add(save(generateDblEntry(getRandomKey()))); + futures.add(save(generateBoolEntry(getRandomKey()))); + } + Futures.allAsList(futures).get(60, TimeUnit.SECONDS); + long endTime = System.currentTimeMillis(); + + long totalTime = endTime - startTime; + + System.out.println("Total time: " + totalTime); + System.out.println("Saved count: " + saveCounter.get()); + System.out.println("Saved per 1 sec: " + saveCounter.get() * 1000 / totalTime); + } + + private void warmup() throws Exception { + List> futures = new ArrayList<>(); + for (int i = 0; i < AMOUNT_OF_UNIQ_KEY; i++) { + futures.add(save(generateStrEntry(i))); + futures.add(save(generateLngEntry(i))); + futures.add(save(generateDblEntry(i))); + futures.add(save(generateBoolEntry(i))); + } + Futures.allAsList(futures).get(60, TimeUnit.SECONDS); + } + + private ListenableFuture save(TsKvEntry tsKvEntry) { + return Futures.transformAsync(testExecutor.submit(() -> timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)), result -> { + saveCounter.incrementAndGet(); + return result; + }, testExecutor); + } + + private TsKvEntry generateStrEntry(int keyIndex) { + return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(STRING_KEY + keyIndex, RandomStringUtils.random(10))); + } + + private TsKvEntry generateLngEntry(int keyIndex) { + return new BasicTsKvEntry(System.currentTimeMillis(), new LongDataEntry(LONG_KEY + keyIndex, random.nextLong())); + } + + private TsKvEntry generateDblEntry(int keyIndex) { + return new BasicTsKvEntry(System.currentTimeMillis(), new DoubleDataEntry(DOUBLE_KEY + keyIndex, random.nextDouble())); + } + + private TsKvEntry generateBoolEntry(int keyIndex) { + return new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(BOOLEAN_KEY + keyIndex, random.nextBoolean())); + } + + private int getRandomKey() { + return random.nextInt(AMOUNT_OF_UNIQ_KEY); + } + +} diff --git a/dao/src/test/resources/logback.xml b/dao/src/test/resources/logback.xml index 5e293b2982..4f7a2142df 100644 --- a/dao/src/test/resources/logback.xml +++ b/dao/src/test/resources/logback.xml @@ -12,6 +12,7 @@ + diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index 9c772df45b..3b1b37242f 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -37,6 +37,7 @@ DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS ts_kv; DROP TABLE IF EXISTS ts_kv_latest; DROP TABLE IF EXISTS ts_kv_dictionary; +DROP SEQUENCE IF EXISTS ts_kv_latest_seq; DROP TABLE IF EXISTS user_credentials; DROP TABLE IF EXISTS widgets_bundle_widget; DROP TABLE IF EXISTS widget_type; From b666f9499ae0d7d41817edce15ff3fd7993dfe3f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 11 Jun 2024 15:41:33 +0200 Subject: [PATCH 18/42] added sequence number for attributes --- .../server/controller/DeviceController.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 2 +- .../dao/timeseries/TimeseriesService.java | 2 +- .../common/data/util/CollectionsUtil.java | 8 + .../dao/AbstractSequenceInsertRepository.java | 128 ++++++++++ .../server/dao/sql/TbSqlBlockingQueue.java | 47 ++-- .../dao/sql/TbSqlBlockingQueueParams.java | 1 + .../dao/sql/TbSqlBlockingQueueWrapper.java | 15 +- .../server/dao/sql/TbSqlQueue.java | 8 +- .../server/dao/sql/TbSqlQueueElement.java | 6 +- .../AttributeKvInsertRepository.java | 226 +++++++----------- .../dao/sql/attributes/JpaAttributeDao.java | 5 +- .../SqlAttributesInsertRepository.java | 27 --- .../dao/sql/edge/JpaBaseEdgeEventDao.java | 2 +- .../server/dao/sql/event/JpaBaseEventDao.java | 2 +- ...stractChunkedAggregationTimeseriesDao.java | 2 +- .../dao/sqlts/SqlTimeseriesLatestDao.java | 36 +-- .../sql/SqlLatestInsertTsRepository.java | 224 ++++++----------- .../timescale/TimescaleTimeseriesDao.java | 8 +- .../dao/timeseries/BaseTimeseriesService.java | 4 +- .../CassandraBaseTimeseriesLatestDao.java | 2 +- .../dao/timeseries/TimeseriesLatestDao.java | 2 +- .../main/resources/sql/schema-entities.sql | 3 + 23 files changed, 382 insertions(+), 382 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/AbstractSequenceInsertRepository.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index fedbd48503..8544f60ca0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -30,6 +31,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -488,7 +490,7 @@ public class DeviceController extends BaseController { @RequestMapping(value = "/devices", params = {"deviceIds"}, method = RequestMethod.GET) @ResponseBody public List getDevicesByIds( - @Parameter(description = "A list of devices ids, separated by comma ','") + @Parameter(description = "A list of devices ids, separated by comma ','", array = @ArraySchema(schema = @Schema(type = "string"))) @RequestParam("deviceIds") String[] strDeviceIds) throws ThingsboardException, ExecutionException, InterruptedException { checkArrayParameter("deviceIds", strDeviceIds); SecurityUser user = getCurrentUser(); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index b19fde2983..3aace70533 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -280,7 +280,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, FutureCallback callback) { - ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); + ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index ffea217da7..eaba144042 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -50,7 +50,7 @@ public interface TimeseriesService { ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); + ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); ListenableFuture> remove(TenantId tenantId, EntityId entityId, List queries); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 4c9006e829..e89f3c1ad8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.util; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -37,6 +38,13 @@ public class CollectionsUtil { return b.stream().filter(p -> !a.contains(p)).collect(Collectors.toSet()); } + /** + * Returns new list with elements that are present in list B(new) but absent in list A(old). + */ + public static List diffLists(List a, List b) { + return b.stream().filter(p -> !a.contains(p)).collect(Collectors.toList()); + } + public static boolean contains(Collection collection, T element) { return isNotEmpty(collection) && collection.contains(element); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/AbstractSequenceInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/AbstractSequenceInsertRepository.java new file mode 100644 index 0000000000..0808f5341f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/AbstractSequenceInsertRepository.java @@ -0,0 +1,128 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.PreparedStatementCreator; +import org.springframework.jdbc.core.SqlProvider; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; +import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public abstract class AbstractSequenceInsertRepository extends AbstractInsertRepository { + + public static final String SEQ_NUMBER = "seq_number"; + + public List saveOrUpdate(List entities) { + return transactionTemplate.execute(status -> { + List seqNumbers = new ArrayList<>(entities.size()); + + KeyHolder keyHolder = new GeneratedKeyHolder(); + + int[] updateResult = onBatchUpdate(entities, keyHolder); + + List> seqNumbersList = keyHolder.getKeyList(); + + int notUpdatedCount = entities.size() - seqNumbersList.size(); + + List toInsertIndexes = new ArrayList<>(notUpdatedCount); + List insertEntities = new ArrayList<>(notUpdatedCount); + int keyHolderIndex = 0; + for (int i = 0; i < updateResult.length; i++) { + if (updateResult[i] == 0) { + insertEntities.add(entities.get(i)); + seqNumbers.add(0L); + toInsertIndexes.add(i); + } else { + seqNumbers.add((Long) seqNumbersList.get(keyHolderIndex).get(SEQ_NUMBER)); + keyHolderIndex++; + } + } + + if (insertEntities.isEmpty()) { + return seqNumbers; + } + + onInsertOrUpdate(insertEntities, keyHolder); + + seqNumbersList = keyHolder.getKeyList(); + + for (int i = 0; i < seqNumbersList.size(); i++) { + seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(SEQ_NUMBER)); + } + + return seqNumbers; + }); + } + + private int[] onBatchUpdate(List entities, KeyHolder keyHolder) { + return jdbcTemplate.batchUpdate(new SequencePreparedStatementCreator(getBatchUpdateQuery()), new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + setOnBatchUpdateValues(ps, i, entities); + } + + @Override + public int getBatchSize() { + return entities.size(); + } + }, keyHolder); + } + + private void onInsertOrUpdate(List insertEntities, KeyHolder keyHolder) { + jdbcTemplate.batchUpdate(new SequencePreparedStatementCreator(getInsertOrUpdateQuery()), new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + setOnInsertOrUpdateValues(ps, i, insertEntities); + } + + @Override + public int getBatchSize() { + return insertEntities.size(); + } + }, keyHolder); + } + + protected abstract void setOnBatchUpdateValues(PreparedStatement ps, int i, List entities) throws SQLException; + + protected abstract void setOnInsertOrUpdateValues(PreparedStatement ps, int i, List entities) throws SQLException; + + protected abstract String getBatchUpdateQuery(); + + protected abstract String getInsertOrUpdateQuery(); + + private record SequencePreparedStatementCreator(String sql) implements PreparedStatementCreator, SqlProvider { + + private static final String[] COLUMNS = {SEQ_NUMBER}; + + @Override + public PreparedStatement createPreparedStatement(Connection con) throws SQLException { + return con.prepareStatement(sql, COLUMNS); + } + + @Override + public String getSql() { + return this.sql; + } + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java index 8f580811a1..68221ad0ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.stats.MessagesStats; import java.util.ArrayList; @@ -29,14 +30,13 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; @Slf4j -public class TbSqlBlockingQueue implements TbSqlQueue { +public class TbSqlBlockingQueue implements TbSqlQueue { - private final BlockingQueue> queue = new LinkedBlockingQueue<>(); + private final BlockingQueue> queue = new LinkedBlockingQueue<>(); private final TbSqlBlockingQueueParams params; private ExecutorService executor; @@ -48,17 +48,17 @@ public class TbSqlBlockingQueue implements TbSqlQueue { } @Override - public void init(ScheduledLogExecutorComponent logExecutor, Consumer> saveFunction, Comparator batchUpdateComparator, int index) { + public void init(ScheduledLogExecutorComponent logExecutor, Function, List> saveFunction, Comparator batchUpdateComparator, Function>, List>> filter, int index) { executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("sql-queue-" + index + "-" + params.getLogName().toLowerCase())); executor.submit(() -> { String logName = params.getLogName(); int batchSize = params.getBatchSize(); long maxDelay = params.getMaxDelay(); - final List> entities = new ArrayList<>(batchSize); + final List> entities = new ArrayList<>(batchSize); while (!Thread.interrupted()) { try { long currentTs = System.currentTimeMillis(); - TbSqlQueueElement attr = queue.poll(maxDelay, TimeUnit.MILLISECONDS); + TbSqlQueueElement attr = queue.poll(maxDelay, TimeUnit.MILLISECONDS); if (attr == null) { continue; } else { @@ -70,12 +70,27 @@ public class TbSqlBlockingQueue implements TbSqlQueue { log.debug("[{}] Going to save {} entities", logName, entities.size()); log.trace("[{}] Going to save entities: {}", logName, entities); } - Stream entitiesStream = entities.stream().map(TbSqlQueueElement::getEntity); - saveFunction.accept( - (params.isBatchSortEnabled() ? entitiesStream.sorted(batchUpdateComparator) : entitiesStream) - .collect(Collectors.toList()) - ); - entities.forEach(v -> v.getFuture().set(null)); + + List> entitiesToSave = filter.apply(entities); + + if (params.isBatchSortEnabled()) { + entitiesToSave = entitiesToSave.stream().sorted((o1, o2) -> batchUpdateComparator.compare(o1.getEntity(), o2.getEntity())).toList(); + } + + List result = saveFunction.apply(entitiesToSave.stream().map(TbSqlQueueElement::getEntity).collect(Collectors.toList())); + + if (params.isWithResponse()) { + for (int i = 0; i < entitiesToSave.size(); i++) { + entitiesToSave.get(i).getFuture().set(result.get(i)); + } + + if (entities.size() > entitiesToSave.size()) { + CollectionsUtil.diffLists(entitiesToSave, entities).forEach(v -> v.getFuture().set(null)); + } + } else { + entities.forEach(v -> v.getFuture().set(null)); + } + stats.incrementSuccessful(entities.size()); if (!fullPack) { long remainingDelay = maxDelay - (System.currentTimeMillis() - currentTs); @@ -104,7 +119,7 @@ public class TbSqlBlockingQueue implements TbSqlQueue { }); logExecutor.scheduleAtFixedRate(() -> { - if (queue.size() > 0 || stats.getTotal() > 0 || stats.getSuccessful() > 0 || stats.getFailed() > 0) { + if (!queue.isEmpty() || stats.getTotal() > 0 || stats.getSuccessful() > 0 || stats.getFailed() > 0) { log.info("Queue-{} [{}] queueSize [{}] totalAdded [{}] totalSaved [{}] totalFailed [{}]", index, params.getLogName(), queue.size(), stats.getTotal(), stats.getSuccessful(), stats.getFailed()); stats.reset(); @@ -120,8 +135,8 @@ public class TbSqlBlockingQueue implements TbSqlQueue { } @Override - public ListenableFuture add(E element) { - SettableFuture future = SettableFuture.create(); + public ListenableFuture add(E element) { + SettableFuture future = SettableFuture.create(); queue.add(new TbSqlQueueElement<>(future, element)); stats.incrementTotal(); return future; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java index 56b7ad3af7..42ab6be049 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java @@ -30,4 +30,5 @@ public class TbSqlBlockingQueueParams { private final long statsPrintIntervalMs; private final String statsNamePrefix; private final boolean batchSortEnabled; + private final boolean withResponse; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueWrapper.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueWrapper.java index 9c5bbbc855..3e9620de76 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueWrapper.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueWrapper.java @@ -29,10 +29,9 @@ import java.util.function.Function; @Slf4j @Data -public class TbSqlBlockingQueueWrapper { - private final CopyOnWriteArrayList> queues = new CopyOnWriteArrayList<>(); +public class TbSqlBlockingQueueWrapper { + private final CopyOnWriteArrayList> queues = new CopyOnWriteArrayList<>(); private final TbSqlBlockingQueueParams params; - private ScheduledLogExecutorComponent logExecutor; private final Function hashCodeFunction; private final int maxThreads; private final StatsFactory statsFactory; @@ -46,15 +45,19 @@ public class TbSqlBlockingQueueWrapper { * NOTE: you must use all of primary key parts in your comparator */ public void init(ScheduledLogExecutorComponent logExecutor, Consumer> saveFunction, Comparator batchUpdateComparator) { + init(logExecutor, l -> { saveFunction.accept(l); return null; }, batchUpdateComparator, l -> l); + } + + public void init(ScheduledLogExecutorComponent logExecutor, Function, List> saveFunction, Comparator batchUpdateComparator, Function>, List>> filter) { for (int i = 0; i < maxThreads; i++) { MessagesStats stats = statsFactory.createMessagesStats(params.getStatsNamePrefix() + ".queue." + i); - TbSqlBlockingQueue queue = new TbSqlBlockingQueue<>(params, stats); + TbSqlBlockingQueue queue = new TbSqlBlockingQueue<>(params, stats); queues.add(queue); - queue.init(logExecutor, saveFunction, batchUpdateComparator, i); + queue.init(logExecutor, saveFunction, batchUpdateComparator, filter, i); } } - public ListenableFuture add(E element) { + public ListenableFuture add(E element) { int queueIndex = element != null ? (hashCodeFunction.apply(element) & 0x7FFFFFFF) % maxThreads : 0; return queues.get(queueIndex).add(element); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueue.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueue.java index 90b4e0fe67..e1ed8c299c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueue.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueue.java @@ -19,13 +19,13 @@ import com.google.common.util.concurrent.ListenableFuture; import java.util.Comparator; import java.util.List; -import java.util.function.Consumer; +import java.util.function.Function; -public interface TbSqlQueue { +public interface TbSqlQueue { - void init(ScheduledLogExecutorComponent logExecutor, Consumer> saveFunction, Comparator batchUpdateComparator, int queueIndex); + void init(ScheduledLogExecutorComponent logExecutor, Function, List> saveFunction, Comparator batchUpdateComparator, Function>, List>> filter, int queueIndex); void destroy(); - ListenableFuture add(E element); + ListenableFuture add(E element); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java index 15031be244..016c5c9527 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java @@ -20,13 +20,13 @@ import lombok.Getter; import lombok.ToString; @ToString(exclude = "future") -public final class TbSqlQueueElement { +public final class TbSqlQueueElement { @Getter - private final SettableFuture future; + private final SettableFuture future; @Getter private final E entity; - public TbSqlQueueElement(SettableFuture future, E entity) { + public TbSqlQueueElement(SettableFuture future, E entity) { this.future = future; this.entity = entity; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 856b2be381..9e188d39b1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -15,162 +15,110 @@ */ package org.thingsboard.server.dao.sql.attributes; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallbackWithoutResult; -import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.dao.AbstractSequenceInsertRepository; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.util.SqlDao; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; -import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; @Repository -@Slf4j +@Transactional @SqlDao -public abstract class AttributeKvInsertRepository { +public class AttributeKvInsertRepository extends AbstractSequenceInsertRepository { - private static final ThreadLocal PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); - private static final String EMPTY_STR = ""; - - private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ? " + - "WHERE entity_id = ? and attribute_type =? and attribute_key = ?;"; + private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, seq_number = nextval('attribute_kv_latest_seq') " + + "WHERE entity_id = ? and attribute_type =? and attribute_key = ? RETURNING seq_number;"; private static final String INSERT_OR_UPDATE = - "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts) " + - "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?) " + + "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts, seq_number) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?, nextval('attribute_kv_latest_seq')) " + "ON CONFLICT (entity_id, attribute_type, attribute_key) " + - "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?;"; - - @Autowired - protected JdbcTemplate jdbcTemplate; - - @Autowired - private TransactionTemplate transactionTemplate; - - @Value("${sql.remove_null_chars:true}") - private boolean removeNullChars; - - public void saveOrUpdate(List entities) { - transactionTemplate.execute(new TransactionCallbackWithoutResult() { - @Override - protected void doInTransactionWithoutResult(TransactionStatus status) { - int[] result = jdbcTemplate.batchUpdate(BATCH_UPDATE, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - AttributeKvEntity kvEntity = entities.get(i); - ps.setString(1, replaceNullChars(kvEntity.getStrValue())); - - if (kvEntity.getLongValue() != null) { - ps.setLong(2, kvEntity.getLongValue()); - } else { - ps.setNull(2, Types.BIGINT); - } - - if (kvEntity.getDoubleValue() != null) { - ps.setDouble(3, kvEntity.getDoubleValue()); - } else { - ps.setNull(3, Types.DOUBLE); - } - - if (kvEntity.getBooleanValue() != null) { - ps.setBoolean(4, kvEntity.getBooleanValue()); - } else { - ps.setNull(4, Types.BOOLEAN); - } - - ps.setString(5, replaceNullChars(kvEntity.getJsonValue())); - - ps.setLong(6, kvEntity.getLastUpdateTs()); - ps.setObject(7, kvEntity.getId().getEntityId()); - ps.setInt(8, kvEntity.getId().getAttributeType()); - ps.setInt(9, kvEntity.getId().getAttributeKey()); - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); - - int updatedCount = 0; - for (int i = 0; i < result.length; i++) { - if (result[i] == 0) { - updatedCount++; - } - } - - List insertEntities = new ArrayList<>(updatedCount); - for (int i = 0; i < result.length; i++) { - if (result[i] == 0) { - insertEntities.add(entities.get(i)); - } - } - - jdbcTemplate.batchUpdate(INSERT_OR_UPDATE, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - AttributeKvEntity kvEntity = insertEntities.get(i); - ps.setObject(1, kvEntity.getId().getEntityId()); - ps.setInt(2, kvEntity.getId().getAttributeType()); - ps.setInt(3, kvEntity.getId().getAttributeKey()); - - ps.setString(4, replaceNullChars(kvEntity.getStrValue())); - ps.setString(10, replaceNullChars(kvEntity.getStrValue())); - - if (kvEntity.getLongValue() != null) { - ps.setLong(5, kvEntity.getLongValue()); - ps.setLong(11, kvEntity.getLongValue()); - } else { - ps.setNull(5, Types.BIGINT); - ps.setNull(11, Types.BIGINT); - } - - if (kvEntity.getDoubleValue() != null) { - ps.setDouble(6, kvEntity.getDoubleValue()); - ps.setDouble(12, kvEntity.getDoubleValue()); - } else { - ps.setNull(6, Types.DOUBLE); - ps.setNull(12, Types.DOUBLE); - } - - if (kvEntity.getBooleanValue() != null) { - ps.setBoolean(7, kvEntity.getBooleanValue()); - ps.setBoolean(13, kvEntity.getBooleanValue()); - } else { - ps.setNull(7, Types.BOOLEAN); - ps.setNull(13, Types.BOOLEAN); - } - - ps.setString(8, replaceNullChars(kvEntity.getJsonValue())); - ps.setString(14, replaceNullChars(kvEntity.getJsonValue())); - - ps.setLong(9, kvEntity.getLastUpdateTs()); - ps.setLong(15, kvEntity.getLastUpdateTs()); - } - - @Override - public int getBatchSize() { - return insertEntities.size(); - } - }); - } - }); + "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, seq_number = nextval('attribute_kv_latest_seq') RETURNING seq_number;"; + + @Override + protected void setOnBatchUpdateValues(PreparedStatement ps, int i, List entities) throws SQLException { + AttributeKvEntity kvEntity = entities.get(i); + ps.setString(1, replaceNullChars(kvEntity.getStrValue())); + + if (kvEntity.getLongValue() != null) { + ps.setLong(2, kvEntity.getLongValue()); + } else { + ps.setNull(2, Types.BIGINT); + } + + if (kvEntity.getDoubleValue() != null) { + ps.setDouble(3, kvEntity.getDoubleValue()); + } else { + ps.setNull(3, Types.DOUBLE); + } + + if (kvEntity.getBooleanValue() != null) { + ps.setBoolean(4, kvEntity.getBooleanValue()); + } else { + ps.setNull(4, Types.BOOLEAN); + } + + ps.setString(5, replaceNullChars(kvEntity.getJsonValue())); + + ps.setLong(6, kvEntity.getLastUpdateTs()); + ps.setObject(7, kvEntity.getId().getEntityId()); + ps.setInt(8, kvEntity.getId().getAttributeType()); + ps.setInt(9, kvEntity.getId().getAttributeKey()); } - private String replaceNullChars(String strValue) { - if (removeNullChars && strValue != null) { - return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); + @Override + protected void setOnInsertOrUpdateValues(PreparedStatement ps, int i, List insertEntities) throws SQLException { + AttributeKvEntity kvEntity = insertEntities.get(i); + ps.setObject(1, kvEntity.getId().getEntityId()); + ps.setInt(2, kvEntity.getId().getAttributeType()); + ps.setInt(3, kvEntity.getId().getAttributeKey()); + + ps.setString(4, replaceNullChars(kvEntity.getStrValue())); + ps.setString(10, replaceNullChars(kvEntity.getStrValue())); + + if (kvEntity.getLongValue() != null) { + ps.setLong(5, kvEntity.getLongValue()); + ps.setLong(11, kvEntity.getLongValue()); + } else { + ps.setNull(5, Types.BIGINT); + ps.setNull(11, Types.BIGINT); } - return strValue; + + if (kvEntity.getDoubleValue() != null) { + ps.setDouble(6, kvEntity.getDoubleValue()); + ps.setDouble(12, kvEntity.getDoubleValue()); + } else { + ps.setNull(6, Types.DOUBLE); + ps.setNull(12, Types.DOUBLE); + } + + if (kvEntity.getBooleanValue() != null) { + ps.setBoolean(7, kvEntity.getBooleanValue()); + ps.setBoolean(13, kvEntity.getBooleanValue()); + } else { + ps.setNull(7, Types.BOOLEAN); + ps.setNull(13, Types.BOOLEAN); + } + + ps.setString(8, replaceNullChars(kvEntity.getJsonValue())); + ps.setString(14, replaceNullChars(kvEntity.getJsonValue())); + + ps.setLong(9, kvEntity.getLastUpdateTs()); + ps.setLong(15, kvEntity.getLastUpdateTs()); + } + + @Override + protected String getBatchUpdateQuery() { + return BATCH_UPDATE; + } + + @Override + protected String getInsertOrUpdateQuery() { + return INSERT_OR_UPDATE; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 320e0e67fb..d8b4a9fedf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -88,7 +88,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Value("${sql.batch_sort:true}") private boolean batchSortEnabled; - private TbSqlBlockingQueueWrapper queue; + private TbSqlBlockingQueueWrapper queue; @PostConstruct private void init() { @@ -99,6 +99,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl .statsPrintIntervalMs(statsPrintIntervalMs) .statsNamePrefix("attributes") .batchSortEnabled(batchSortEnabled) + .withResponse(true) .build(); Function hashcodeFunction = entity -> entity.getId().getEntityId().hashCode(); @@ -106,7 +107,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl queue.init(logExecutor, v -> attributeKvInsertRepository.saveOrUpdate(v), Comparator.comparing((AttributeKvEntity attributeKvEntity) -> attributeKvEntity.getId().getEntityId()) .thenComparing(attributeKvEntity -> attributeKvEntity.getId().getAttributeType()) - .thenComparing(attributeKvEntity -> attributeKvEntity.getId().getAttributeKey()) + .thenComparing(attributeKvEntity -> attributeKvEntity.getId().getAttributeKey()), l -> l ); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java deleted file mode 100644 index a99f5a24e7..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.attributes; - -import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.util.SqlDao; - -@Repository -@Transactional -@SqlDao -public class SqlAttributesInsertRepository extends AttributeKvInsertRepository { - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index 1c64973d94..d9d2282fca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -90,7 +90,7 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao queue; + private TbSqlBlockingQueueWrapper queue; @Override protected Class getEntityClass() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index afd6608774..33e7567664 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -110,7 +110,7 @@ public class JpaBaseEventDao implements EventDao { @Value("${sql.batch_sort:true}") private boolean batchSortEnabled; - private TbSqlBlockingQueueWrapper queue; + private TbSqlBlockingQueueWrapper queue; private final Map> repositories = new ConcurrentHashMap<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 9a52f3a192..f3b7b96c87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -60,7 +60,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @Autowired protected InsertTsRepository insertRepository; - protected TbSqlBlockingQueueWrapper tsQueue; + protected TbSqlBlockingQueueWrapper tsQueue; @Autowired private StatsFactory statsFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 425bb10a0e..513414b8ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -46,6 +46,7 @@ import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sql.TbSqlQueueElement; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; @@ -81,7 +82,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme @Autowired private InsertLatestTsRepository insertLatestTsRepository; - private TbSqlBlockingQueueWrapper tsLatestQueue; + private TbSqlBlockingQueueWrapper tsLatestQueue; @Value("${sql.ts_latest.batch_size:1000}") private int tsLatestBatchSize; @@ -115,25 +116,26 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme .maxDelay(tsLatestMaxDelay) .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) .statsNamePrefix("ts.latest") - .batchSortEnabled(false) + .batchSortEnabled(batchSortEnabled) + .withResponse(true) .build(); java.util.function.Function hashcodeFunction = entity -> entity.getEntityId().hashCode(); tsLatestQueue = new TbSqlBlockingQueueWrapper<>(tsLatestParams, hashcodeFunction, tsLatestBatchThreads, statsFactory); - tsLatestQueue.init(logExecutor, v -> { - Map trueLatest = new HashMap<>(); - v.forEach(ts -> { - TsKey key = new TsKey(ts.getEntityId(), ts.getKey()); - trueLatest.merge(key, ts, (oldTs, newTs) -> oldTs.getTs() <= newTs.getTs() ? newTs : oldTs); - }); - List latestEntities = new ArrayList<>(trueLatest.values()); - if (batchSortEnabled) { - latestEntities.sort(Comparator.comparing((Function) AbstractTsKvEntity::getEntityId) - .thenComparingInt(AbstractTsKvEntity::getKey)); - } - insertLatestTsRepository.saveOrUpdate(latestEntities); - }, (l, r) -> 0); + tsLatestQueue.init(logExecutor, + v -> insertLatestTsRepository.saveOrUpdate(v), + Comparator.comparing((Function) AbstractTsKvEntity::getEntityId) + .thenComparingInt(AbstractTsKvEntity::getKey), + v -> { + Map> trueLatest = new HashMap<>(); + v.forEach(element -> { + var entity = element.getEntity(); + TsKey key = new TsKey(entity.getEntityId(), entity.getKey()); + trueLatest.merge(key, element, (oldElement, newElement) -> oldElement.getEntity().getTs() <= newElement.getEntity().getTs() ? newElement : oldElement); + }); + return new ArrayList<>(trueLatest.values()); + }); } @PreDestroy @@ -144,7 +146,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme } @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { return getSaveLatestFuture(entityId, tsKvEntry); } @@ -247,7 +249,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme searchTsKvLatestRepository.findAllByEntityId(entityId.getId())))); } - protected ListenableFuture getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { + protected ListenableFuture getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { TsKvLatestEntity latestEntity = new TsKvLatestEntity(); latestEntity.setEntityId(entityId.getId()); latestEntity.setTs(tsKvEntry.getTs()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index 48b54c2281..5b953dfe59 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -17,33 +17,25 @@ package org.thingsboard.server.dao.sqlts.insert.latest.sql; import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; -import org.springframework.jdbc.core.BatchPreparedStatementSetter; -import org.springframework.jdbc.core.PreparedStatementCreator; -import org.springframework.jdbc.core.SqlProvider; -import org.springframework.jdbc.support.GeneratedKeyHolder; -import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.dao.AbstractSequenceInsertRepository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; -import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; -import java.util.ArrayList; import java.util.List; -import java.util.Map; @SqlTsLatestAnyDao @Repository @Transactional @SqlDao -public class SqlLatestInsertTsRepository extends AbstractInsertRepository implements InsertLatestTsRepository { +public class SqlLatestInsertTsRepository extends AbstractSequenceInsertRepository implements InsertLatestTsRepository { @Value("${sql.ts_latest.update_by_latest_ts:true}") private Boolean updateByLatestTs; @@ -61,8 +53,6 @@ public class SqlLatestInsertTsRepository extends AbstractInsertRepository implem private static final String RETURNING = " RETURNING seq_number"; - private static final String SEQ_NUMBER = "seq_number"; - private String batchUpdateQuery; private String insertOrUpdateQuery; @@ -73,161 +63,89 @@ public class SqlLatestInsertTsRepository extends AbstractInsertRepository implem } @Override - public List saveOrUpdate(List entities) { - return transactionTemplate.execute(status -> { - List seqNumbers = new ArrayList<>(entities.size()); + protected void setOnBatchUpdateValues(PreparedStatement ps, int i, List entities) throws SQLException { + TsKvLatestEntity tsKvLatestEntity = entities.get(i); + ps.setLong(1, tsKvLatestEntity.getTs()); + + if (tsKvLatestEntity.getBooleanValue() != null) { + ps.setBoolean(2, tsKvLatestEntity.getBooleanValue()); + } else { + ps.setNull(2, Types.BOOLEAN); + } - KeyHolder keyHolder = new GeneratedKeyHolder(); + ps.setString(3, replaceNullChars(tsKvLatestEntity.getStrValue())); - int[] updateResult = onBatchUpdate(entities, keyHolder); + if (tsKvLatestEntity.getLongValue() != null) { + ps.setLong(4, tsKvLatestEntity.getLongValue()); + } else { + ps.setNull(4, Types.BIGINT); + } - List> seqNumbersList = keyHolder.getKeyList(); + if (tsKvLatestEntity.getDoubleValue() != null) { + ps.setDouble(5, tsKvLatestEntity.getDoubleValue()); + } else { + ps.setNull(5, Types.DOUBLE); + } - int notUpdatedCount = entities.size() - seqNumbersList.size(); + ps.setString(6, replaceNullChars(tsKvLatestEntity.getJsonValue())); - List toInsertIndexes = new ArrayList<>(notUpdatedCount); - List insertEntities = new ArrayList<>(notUpdatedCount); - int keyHolderIndex = 0; - for (int i = 0; i < updateResult.length; i++) { - if (updateResult[i] == 0) { - insertEntities.add(entities.get(i)); - seqNumbers.add(0L); - toInsertIndexes.add(i); - } else { - seqNumbers.add((Long) seqNumbersList.get(keyHolderIndex).get(SEQ_NUMBER)); - keyHolderIndex++; - } - } + ps.setObject(7, tsKvLatestEntity.getEntityId()); + ps.setInt(8, tsKvLatestEntity.getKey()); + if (updateByLatestTs) { + ps.setLong(9, tsKvLatestEntity.getTs()); + } + } - if (insertEntities.isEmpty()) { - return seqNumbers; - } + @Override + protected void setOnInsertOrUpdateValues(PreparedStatement ps, int i, List insertEntities) throws SQLException { + TsKvLatestEntity tsKvLatestEntity = insertEntities.get(i); + ps.setObject(1, tsKvLatestEntity.getEntityId()); + ps.setInt(2, tsKvLatestEntity.getKey()); + + ps.setLong(3, tsKvLatestEntity.getTs()); + ps.setLong(9, tsKvLatestEntity.getTs()); + if (updateByLatestTs) { + ps.setLong(15, tsKvLatestEntity.getTs()); + } - onInsertOrUpdate(insertEntities, keyHolder); + if (tsKvLatestEntity.getBooleanValue() != null) { + ps.setBoolean(4, tsKvLatestEntity.getBooleanValue()); + ps.setBoolean(10, tsKvLatestEntity.getBooleanValue()); + } else { + ps.setNull(4, Types.BOOLEAN); + ps.setNull(10, Types.BOOLEAN); + } - seqNumbersList = keyHolder.getKeyList(); + ps.setString(5, replaceNullChars(tsKvLatestEntity.getStrValue())); + ps.setString(11, replaceNullChars(tsKvLatestEntity.getStrValue())); - for (int i = 0; i < seqNumbersList.size(); i++) { - seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(SEQ_NUMBER)); - } + if (tsKvLatestEntity.getLongValue() != null) { + ps.setLong(6, tsKvLatestEntity.getLongValue()); + ps.setLong(12, tsKvLatestEntity.getLongValue()); + } else { + ps.setNull(6, Types.BIGINT); + ps.setNull(12, Types.BIGINT); + } - return seqNumbers; - }); - } + if (tsKvLatestEntity.getDoubleValue() != null) { + ps.setDouble(7, tsKvLatestEntity.getDoubleValue()); + ps.setDouble(13, tsKvLatestEntity.getDoubleValue()); + } else { + ps.setNull(7, Types.DOUBLE); + ps.setNull(13, Types.DOUBLE); + } - private int[] onBatchUpdate(List entities, KeyHolder keyHolder) { - return jdbcTemplate.batchUpdate(new SimplePreparedStatementCreator(batchUpdateQuery), new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - TsKvLatestEntity tsKvLatestEntity = entities.get(i); - ps.setLong(1, tsKvLatestEntity.getTs()); - - if (tsKvLatestEntity.getBooleanValue() != null) { - ps.setBoolean(2, tsKvLatestEntity.getBooleanValue()); - } else { - ps.setNull(2, Types.BOOLEAN); - } - - ps.setString(3, replaceNullChars(tsKvLatestEntity.getStrValue())); - - if (tsKvLatestEntity.getLongValue() != null) { - ps.setLong(4, tsKvLatestEntity.getLongValue()); - } else { - ps.setNull(4, Types.BIGINT); - } - - if (tsKvLatestEntity.getDoubleValue() != null) { - ps.setDouble(5, tsKvLatestEntity.getDoubleValue()); - } else { - ps.setNull(5, Types.DOUBLE); - } - - ps.setString(6, replaceNullChars(tsKvLatestEntity.getJsonValue())); - - ps.setObject(7, tsKvLatestEntity.getEntityId()); - ps.setInt(8, tsKvLatestEntity.getKey()); - if (updateByLatestTs) { - ps.setLong(9, tsKvLatestEntity.getTs()); - } - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }, keyHolder); + ps.setString(8, replaceNullChars(tsKvLatestEntity.getJsonValue())); + ps.setString(14, replaceNullChars(tsKvLatestEntity.getJsonValue())); } - private void onInsertOrUpdate(List insertEntities, KeyHolder keyHolder) { - jdbcTemplate.batchUpdate(new SimplePreparedStatementCreator(insertOrUpdateQuery), new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - TsKvLatestEntity tsKvLatestEntity = insertEntities.get(i); - ps.setObject(1, tsKvLatestEntity.getEntityId()); - ps.setInt(2, tsKvLatestEntity.getKey()); - - ps.setLong(3, tsKvLatestEntity.getTs()); - ps.setLong(9, tsKvLatestEntity.getTs()); - if (updateByLatestTs) { - ps.setLong(15, tsKvLatestEntity.getTs()); - } - - if (tsKvLatestEntity.getBooleanValue() != null) { - ps.setBoolean(4, tsKvLatestEntity.getBooleanValue()); - ps.setBoolean(10, tsKvLatestEntity.getBooleanValue()); - } else { - ps.setNull(4, Types.BOOLEAN); - ps.setNull(10, Types.BOOLEAN); - } - - ps.setString(5, replaceNullChars(tsKvLatestEntity.getStrValue())); - ps.setString(11, replaceNullChars(tsKvLatestEntity.getStrValue())); - - if (tsKvLatestEntity.getLongValue() != null) { - ps.setLong(6, tsKvLatestEntity.getLongValue()); - ps.setLong(12, tsKvLatestEntity.getLongValue()); - } else { - ps.setNull(6, Types.BIGINT); - ps.setNull(12, Types.BIGINT); - } - - if (tsKvLatestEntity.getDoubleValue() != null) { - ps.setDouble(7, tsKvLatestEntity.getDoubleValue()); - ps.setDouble(13, tsKvLatestEntity.getDoubleValue()); - } else { - ps.setNull(7, Types.DOUBLE); - ps.setNull(13, Types.DOUBLE); - } - - ps.setString(8, replaceNullChars(tsKvLatestEntity.getJsonValue())); - ps.setString(14, replaceNullChars(tsKvLatestEntity.getJsonValue())); - } - - @Override - public int getBatchSize() { - return insertEntities.size(); - } - }, keyHolder); + @Override + protected String getBatchUpdateQuery() { + return batchUpdateQuery; } - private static class SimplePreparedStatementCreator implements PreparedStatementCreator, SqlProvider { - - private static final String[] COLUMNS = {SEQ_NUMBER}; - private final String sql; - - - public SimplePreparedStatementCreator(String sql) { - this.sql = sql; - } - - @Override - public PreparedStatement createPreparedStatement(Connection con) throws SQLException { - return con.prepareStatement(sql, COLUMNS); - } - - @Override - public String getSql() { - return this.sql; - } + @Override + protected String getInsertOrUpdateQuery() { + return insertOrUpdateQuery; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index c8bff8ceb9..cd9f397bb3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -18,8 +18,9 @@ package org.thingsboard.server.dao.sqlts.timescale; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; @@ -38,7 +39,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; @@ -47,8 +47,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.dao.util.TimeUtils; import org.thingsboard.server.dao.util.TimescaleDBTsDao; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -77,7 +75,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @Autowired protected KeyDictionaryDao keyDictionaryDao; - protected TbSqlBlockingQueueWrapper tsQueue; + protected TbSqlBlockingQueueWrapper tsQueue; @PostConstruct protected void init() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 60056b2b8f..f9c4218d64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -187,8 +187,8 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { - List> futures = new ArrayList<>(tsKvEntries.size()); + public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { + List> futures = new ArrayList<>(tsKvEntries.size()); for (TsKvEntry tsKvEntry : tsKvEntries) { futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 7a5904eb6b..f774f18dc7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -100,7 +100,7 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes } @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); stmtBuilder.setString(0, entityId.getEntityType().name()) .setUuid(1, entityId.getId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index d339f49e11..9f62fd033a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -42,7 +42,7 @@ public interface TimeseriesLatestDao { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index b92c27574c..a26262970c 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -102,6 +102,8 @@ CREATE TABLE IF NOT EXISTS audit_log ( action_failure_details varchar(1000000) ) PARTITION BY RANGE (created_time); +CREATE SEQUENCE IF NOT EXISTS attribute_kv_latest_seq cache 1000; + CREATE TABLE IF NOT EXISTS attribute_kv ( entity_id uuid, attribute_type int, @@ -112,6 +114,7 @@ CREATE TABLE IF NOT EXISTS attribute_kv ( dbl_v double precision, json_v json, last_update_ts bigint, + seq_number bigint, CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); From 4807a7cbdd9156da4f9912fd51f30e8d40bc8fca Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 11 Jun 2024 17:15:55 +0200 Subject: [PATCH 19/42] ranamed seq_number to version and refactoring --- .../server/controller/DeviceController.java | 4 +--- ...java => AbstractVersionedInsertRepository.java} | 10 +++++----- .../attributes/AttributeKvInsertRepository.java | 14 +++++++------- .../latest/sql/SqlLatestInsertTsRepository.java | 12 ++++++------ dao/src/main/resources/sql/schema-entities.sql | 8 ++++---- dao/src/main/resources/sql/schema-timescale.sql | 3 +++ .../main/resources/sql/schema-ts-latest-psql.sql | 10 +++------- .../test/resources/sql/psql/drop-all-tables.sql | 3 ++- 8 files changed, 31 insertions(+), 33 deletions(-) rename dao/src/main/java/org/thingsboard/server/dao/{AbstractSequenceInsertRepository.java => AbstractVersionedInsertRepository.java} (93%) diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 8544f60ca0..fedbd48503 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; @@ -31,7 +30,6 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -490,7 +488,7 @@ public class DeviceController extends BaseController { @RequestMapping(value = "/devices", params = {"deviceIds"}, method = RequestMethod.GET) @ResponseBody public List getDevicesByIds( - @Parameter(description = "A list of devices ids, separated by comma ','", array = @ArraySchema(schema = @Schema(type = "string"))) + @Parameter(description = "A list of devices ids, separated by comma ','") @RequestParam("deviceIds") String[] strDeviceIds) throws ThingsboardException, ExecutionException, InterruptedException { checkArrayParameter("deviceIds", strDeviceIds); SecurityUser user = getCurrentUser(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/AbstractSequenceInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java similarity index 93% rename from dao/src/main/java/org/thingsboard/server/dao/AbstractSequenceInsertRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java index 0808f5341f..cb92ca5e1f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/AbstractSequenceInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java @@ -29,9 +29,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -public abstract class AbstractSequenceInsertRepository extends AbstractInsertRepository { +public abstract class AbstractVersionedInsertRepository extends AbstractInsertRepository { - public static final String SEQ_NUMBER = "seq_number"; + public static final String VERSION_COLUMN = "version"; public List saveOrUpdate(List entities) { return transactionTemplate.execute(status -> { @@ -54,7 +54,7 @@ public abstract class AbstractSequenceInsertRepository extends AbstractInsert seqNumbers.add(0L); toInsertIndexes.add(i); } else { - seqNumbers.add((Long) seqNumbersList.get(keyHolderIndex).get(SEQ_NUMBER)); + seqNumbers.add((Long) seqNumbersList.get(keyHolderIndex).get(VERSION_COLUMN)); keyHolderIndex++; } } @@ -68,7 +68,7 @@ public abstract class AbstractSequenceInsertRepository extends AbstractInsert seqNumbersList = keyHolder.getKeyList(); for (int i = 0; i < seqNumbersList.size(); i++) { - seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(SEQ_NUMBER)); + seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(VERSION_COLUMN)); } return seqNumbers; @@ -113,7 +113,7 @@ public abstract class AbstractSequenceInsertRepository extends AbstractInsert private record SequencePreparedStatementCreator(String sql) implements PreparedStatementCreator, SqlProvider { - private static final String[] COLUMNS = {SEQ_NUMBER}; + private static final String[] COLUMNS = {VERSION_COLUMN}; @Override public PreparedStatement createPreparedStatement(Connection con) throws SQLException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 9e188d39b1..586ee3c0ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -17,7 +17,7 @@ package org.thingsboard.server.dao.sql.attributes; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.AbstractSequenceInsertRepository; +import org.thingsboard.server.dao.AbstractVersionedInsertRepository; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.util.SqlDao; @@ -29,16 +29,16 @@ import java.util.List; @Repository @Transactional @SqlDao -public class AttributeKvInsertRepository extends AbstractSequenceInsertRepository { +public class AttributeKvInsertRepository extends AbstractVersionedInsertRepository { - private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, seq_number = nextval('attribute_kv_latest_seq') " + - "WHERE entity_id = ? and attribute_type =? and attribute_key = ? RETURNING seq_number;"; + private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') " + + "WHERE entity_id = ? and attribute_type =? and attribute_key = ? RETURNING version;"; private static final String INSERT_OR_UPDATE = - "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts, seq_number) " + - "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?, nextval('attribute_kv_latest_seq')) " + + "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts, version) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?, nextval('attribute_kv_version_seq')) " + "ON CONFLICT (entity_id, attribute_type, attribute_key) " + - "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, seq_number = nextval('attribute_kv_latest_seq') RETURNING seq_number;"; + "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') RETURNING version;"; @Override protected void setOnBatchUpdateValues(PreparedStatement ps, int i, List entities) throws SQLException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index 5b953dfe59..323eec0851 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -19,7 +19,7 @@ import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.dao.AbstractSequenceInsertRepository; +import org.thingsboard.server.dao.AbstractVersionedInsertRepository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.util.SqlDao; @@ -35,23 +35,23 @@ import java.util.List; @Repository @Transactional @SqlDao -public class SqlLatestInsertTsRepository extends AbstractSequenceInsertRepository implements InsertLatestTsRepository { +public class SqlLatestInsertTsRepository extends AbstractVersionedInsertRepository implements InsertLatestTsRepository { @Value("${sql.ts_latest.update_by_latest_ts:true}") private Boolean updateByLatestTs; private static final String BATCH_UPDATE = - "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), seq_number = nextval('ts_kv_latest_seq') WHERE entity_id = ? AND key = ?"; + "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = nextval('ts_kv_latest_version_seq') WHERE entity_id = ? AND key = ?"; private static final String INSERT_OR_UPDATE = - "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, seq_number) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), nextval('ts_kv_latest_seq')) " + - "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), seq_number = nextval('ts_kv_latest_seq')"; + "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), nextval('ts_kv_latest_version_seq')) " + + "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = nextval('ts_kv_latest_version_seq')"; private static final String BATCH_UPDATE_BY_LATEST_TS = BATCH_UPDATE + " AND ts_kv_latest.ts <= ?"; private static final String INSERT_OR_UPDATE_BY_LATEST_TS = INSERT_OR_UPDATE + " WHERE ts_kv_latest.ts <= ?"; - private static final String RETURNING = " RETURNING seq_number"; + private static final String RETURNING = " RETURNING version"; private String batchUpdateQuery; private String insertOrUpdateQuery; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a26262970c..45deaf931d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -102,7 +102,7 @@ CREATE TABLE IF NOT EXISTS audit_log ( action_failure_details varchar(1000000) ) PARTITION BY RANGE (created_time); -CREATE SEQUENCE IF NOT EXISTS attribute_kv_latest_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1000; CREATE TABLE IF NOT EXISTS attribute_kv ( entity_id uuid, @@ -114,7 +114,7 @@ CREATE TABLE IF NOT EXISTS attribute_kv ( dbl_v double precision, json_v json, last_update_ts bigint, - seq_number bigint, + version bigint, CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); @@ -541,7 +541,7 @@ CREATE TABLE IF NOT EXISTS entity_view ( CONSTRAINT entity_view_external_id_unq_key UNIQUE (tenant_id, external_id) ); -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; CREATE TABLE IF NOT EXISTS ts_kv_latest ( @@ -553,7 +553,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, - seq_number bigint, + version bigint, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index ecd8aa1e9d..e5d44cb24a 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -34,6 +34,8 @@ CREATE TABLE IF NOT EXISTS key_dictionary ( CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; + CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, key int NOT NULL, @@ -43,6 +45,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest ( long_v bigint, dbl_v double precision, json_v json, + version bigint, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-ts-latest-psql.sql b/dao/src/main/resources/sql/schema-ts-latest-psql.sql index c6701315b4..adae100823 100644 --- a/dao/src/main/resources/sql/schema-ts-latest-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-latest-psql.sql @@ -14,6 +14,8 @@ -- limitations under the License. -- +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; + CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, @@ -24,12 +26,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, + version bigint, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); - -CREATE TABLE IF NOT EXISTS ts_kv_dictionary -( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) ); \ No newline at end of file diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index 3b1b37242f..c731e239af 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -23,6 +23,7 @@ DROP TABLE IF EXISTS alarm_type; DROP TABLE IF EXISTS asset; DROP TABLE IF EXISTS audit_log; DROP TABLE IF EXISTS attribute_kv; +DROP SEQUENCE IF EXISTS attribute_kv_version_seq; DROP TABLE IF EXISTS component_descriptor; DROP TABLE IF EXISTS customer; DROP TABLE IF EXISTS device; @@ -37,7 +38,7 @@ DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS ts_kv; DROP TABLE IF EXISTS ts_kv_latest; DROP TABLE IF EXISTS ts_kv_dictionary; -DROP SEQUENCE IF EXISTS ts_kv_latest_seq; +DROP SEQUENCE IF EXISTS ts_kv_latest_version_seq; DROP TABLE IF EXISTS user_credentials; DROP TABLE IF EXISTS widgets_bundle_widget; DROP TABLE IF EXISTS widget_type; From e2be5c6492c2baab95200764da93ac595d206324 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 12 Jun 2024 12:23:19 +0200 Subject: [PATCH 20/42] return version instead of key for attributes --- .../device/DeviceProvisionServiceImpl.java | 2 +- .../service/edge/rpc/EdgeGrpcSession.java | 6 +-- .../DefaultSystemDataLoaderService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 4 +- .../dao/attributes/AttributesService.java | 22 ++------ .../server/dao/attributes/AttributesDao.java | 2 +- .../dao/attributes/BaseAttributesService.java | 42 ++-------------- .../attributes/CachedAttributesService.java | 50 +++++-------------- .../dao/sql/attributes/JpaAttributeDao.java | 10 ++-- .../server/dao/service/EntityServiceTest.java | 16 +++--- .../sql/LatestTimeseriesPerformanceTest.java | 8 +-- 11 files changed, 45 insertions(+), 119 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index bedf9c7537..f5b6411bf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -240,7 +240,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { return deviceCredentialsService.updateDeviceCredentials(tenantId, deviceCredentials); } - private ListenableFuture> saveProvisionStateAttribute(Device device) { + private ListenableFuture> saveProvisionStateAttribute(Device device) { return attributesService.save(device.getTenantId(), device.getId(), AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), System.currentTimeMillis()))); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 55e8e051ab..81f90fee51 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -371,10 +371,10 @@ public final class EdgeGrpcSession implements Closeable { @Override public void onSuccess(@Nullable Pair newStartTsAndSeqId) { if (newStartTsAndSeqId != null) { - ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); + ListenableFuture> updateFuture = updateQueueStartTsAndSeqId(newStartTsAndSeqId); Futures.addCallback(updateFuture, new FutureCallback<>() { @Override - public void onSuccess(@Nullable List list) { + public void onSuccess(@Nullable List list) { log.debug("[{}][{}] queue offset was updated [{}]", tenantId, sessionId, newStartTsAndSeqId); if (fetcher.isSeqIdNewCycleStarted()) { seqIdEnd = fetcher.getSeqIdEnd(); @@ -626,7 +626,7 @@ public final class EdgeGrpcSession implements Closeable { return startSeqId; } - private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { + private ListenableFuture> updateQueueStartTsAndSeqId(Pair pair) { this.newStartTs = pair.getFirst(); this.newStartSeqId = pair.getSecond(); log.trace("[{}] updateQueueStartTsAndSeqId [{}][{}][{}]", this.sessionId, edge.getId(), this.newStartTs, this.newStartSeqId); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 7588726a73..e49a42b57b 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -577,7 +577,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } else { - ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, + ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) , System.currentTimeMillis()))); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 3aace70533..70962e821a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -260,14 +260,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { - ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); + ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes, boolean notifyDevice, FutureCallback callback) { - ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); + ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index d869471fa3..a205ee9b84 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -32,30 +31,18 @@ import java.util.Optional; */ public interface AttributesService { - @Deprecated(since = "3.7.0") - ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey); - ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey); - @Deprecated(since = "3.7.0") - ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys); - ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, Collection attributeKeys); - @Deprecated(since = "3.7.0") - ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope); - ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope); @Deprecated(since = "3.7.0") - ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes); - - ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); + ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes); - @Deprecated(since = "3.7.0") - ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute); + ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes); - ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute); @Deprecated(since = "3.7.0") ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys); @@ -64,9 +51,6 @@ public interface AttributesService { List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); - @Deprecated(since = "3.7.0") - List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds); - List findAllKeysByEntityIds(TenantId tenantId, List entityIds); List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 64ca5d9dd9..34f67fb5ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -38,7 +38,7 @@ public interface AttributesDao { List findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope); - ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute); List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 1b1f2d005d..6f49b2a94e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -56,13 +56,6 @@ public class BaseAttributesService implements AttributesService { this.attributesDao = attributesDao; } - @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { - validate(entityId, scope); - Validator.validateString(attributeKey, k -> "Incorrect attribute key " + k); - return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey)); - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -70,13 +63,6 @@ public class BaseAttributesService implements AttributesService { return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKey)); } - @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { - validate(entityId, scope); - attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, k -> "Incorrect attribute key " + k)); - return Futures.immediateFuture(attributesDao.find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeys)); - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, Collection attributeKeys) { validate(entityId, scope); @@ -84,12 +70,6 @@ public class BaseAttributesService implements AttributesService { return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKeys)); } - @Override - public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { - validate(entityId, scope); - return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, AttributeScope.valueOf(scope))); - } - @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope) { validate(entityId, scope); @@ -101,11 +81,6 @@ public class BaseAttributesService implements AttributesService { return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId); } - @Override - public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { - return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); - } - @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); @@ -121,32 +96,25 @@ public class BaseAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { - validate(entityId, scope); - AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); - } - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); return attributesDao.save(tenantId, entityId, scope, attribute); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, AttributeScope.valueOf(scope), attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index f51bc6f0b8..375b56b1f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -29,7 +29,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -109,12 +108,6 @@ public class CachedAttributesService implements AttributesService { return cacheExecutorService.executor(); } - - @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { - return find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKey); - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -144,16 +137,11 @@ public class CachedAttributesService implements AttributesService { }); } - @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, final Collection attributeKeysNonUnique) { - return find(tenantId, entityId, AttributeScope.valueOf(scope), attributeKeysNonUnique); - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, final Collection attributeKeysNonUnique) { validate(entityId, scope); final var attributeKeys = new LinkedHashSet<>(attributeKeysNonUnique); // deduplicate the attributes - attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, k ->"Incorrect attribute key " + k)); + attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, k -> "Incorrect attribute key " + k)); //CacheExecutor for Redis or DirectExecutor for local Caffeine return Futures.transformAsync(cacheExecutor.submit(() -> findCachedAttributes(entityId, scope, attributeKeys)), @@ -216,11 +204,6 @@ public class CachedAttributesService implements AttributesService { return cachedAttributes; } - @Override - public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { - return findAll(tenantId, entityId, AttributeScope.valueOf(scope)); - } - @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, AttributeScope scope) { validate(entityId, scope); @@ -233,11 +216,6 @@ public class CachedAttributesService implements AttributesService { return attributesDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId); } - @Override - public List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds) { - return findAllKeysByEntityIds(tenantId, entityIds); - } - @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); @@ -253,42 +231,38 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { - return save(tenantId, entityId, AttributeScope.valueOf(scope), attribute); - } - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor); + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + return Futures.transform(future, version -> evict(entityId, scope, attribute, version), cacheExecutor); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { return save(tenantId, entityId, AttributeScope.valueOf(scope), attributes); } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> futures = new ArrayList<>(attributes.size()); + List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - futures.add(Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor)); + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + futures.add(Futures.transform(future, version -> evict(entityId, scope, attribute, version), cacheExecutor)); } return Futures.allAsList(futures); } - private String evict(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute, String key) { + private Long evict(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute, Long version) { + String key = attribute.getKey(); log.trace("[{}][{}][{}] Before cache evict: {}", entityId, scope, key, attribute); cache.evictOrPut(new AttributeCacheKey(scope, entityId, key), attribute); log.trace("[{}][{}][{}] after cache evict.", entityId, scope, key); - return key; + return version; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index d8b4a9fedf..05cdfce221 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -16,9 +16,7 @@ package org.thingsboard.server.dao.sql.attributes; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; @@ -179,7 +177,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute) { AttributeKvEntity entity = new AttributeKvEntity(); entity.setId(new AttributeKvCompositeKey(entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(attribute.getKey()))); entity.setLastUpdateTs(attribute.getLastUpdateTs()); @@ -188,11 +186,11 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl entity.setLongValue(attribute.getLongValue().orElse(null)); entity.setBooleanValue(attribute.getBooleanValue().orElse(null)); entity.setJsonValue(attribute.getJsonValue().orElse(null)); - return addToQueue(entity, attribute.getKey()); + return addToQueue(entity); } - private ListenableFuture addToQueue(AttributeKvEntity entity, String key) { - return Futures.transform(queue.add(entity), v -> key, MoreExecutors.directExecutor()); + private ListenableFuture addToQueue(AttributeKvEntity entity) { + return queue.add(entity); } @Override diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java index 1fc2ff2ce6..6c87b0128f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java @@ -412,7 +412,7 @@ public class EntityServiceTest extends AbstractServiceTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -591,7 +591,7 @@ public class EntityServiceTest extends AbstractServiceTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -666,7 +666,7 @@ public class EntityServiceTest extends AbstractServiceTest { List highConsumptions = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { Asset asset = assets.get(i); attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), AttributeScope.SERVER_SCOPE)); @@ -1586,7 +1586,7 @@ public class EntityServiceTest extends AbstractServiceTest { } } - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); for (AttributeScope currentScope : AttributeScope.values()) { @@ -1688,7 +1688,7 @@ public class EntityServiceTest extends AbstractServiceTest { } } - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), AttributeScope.CLIENT_SCOPE)); @@ -1966,7 +1966,7 @@ public class EntityServiceTest extends AbstractServiceTest { } } - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveStringAttribute(device.getId(), "attributeString", attributeStrings.get(i), AttributeScope.CLIENT_SCOPE)); @@ -2428,13 +2428,13 @@ public class EntityServiceTest extends AbstractServiceTest { return filter; } - private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { + private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { + private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java index 00f458ca52..d81eea997c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/sql/LatestTimeseriesPerformanceTest.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Assert; @@ -48,6 +49,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @DaoSqlTest +@Slf4j public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { private static final String STRING_KEY = "stringKey"; @@ -105,9 +107,9 @@ public class LatestTimeseriesPerformanceTest extends AbstractServiceTest { long totalTime = endTime - startTime; - System.out.println("Total time: " + totalTime); - System.out.println("Saved count: " + saveCounter.get()); - System.out.println("Saved per 1 sec: " + saveCounter.get() * 1000 / totalTime); + log.info("Total time: {}", totalTime); + log.info("Saved count: {}", saveCounter.get()); + log.warn("Saved per 1 sec: {}", saveCounter.get() * 1000 / totalTime); } private void warmup() throws Exception { From e57b9471dcac00a3b30e3f75702061b50c772e66 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 13 Jun 2024 19:18:01 +0200 Subject: [PATCH 21/42] used version + 1 instead of sequence next value --- .../sql/attributes/AttributeKvInsertRepository.java | 10 +++++----- .../insert/latest/sql/SqlLatestInsertTsRepository.java | 7 +++---- dao/src/main/resources/sql/schema-entities.sql | 8 ++------ dao/src/main/resources/sql/schema-timescale.sql | 4 +--- dao/src/main/resources/sql/schema-ts-latest-psql.sql | 4 +--- dao/src/test/resources/sql/psql/drop-all-tables.sql | 2 -- 6 files changed, 12 insertions(+), 23 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 586ee3c0ca..c2b9d53d47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -31,14 +31,14 @@ import java.util.List; @SqlDao public class AttributeKvInsertRepository extends AbstractVersionedInsertRepository { - private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') " + + private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = version + 1 " + "WHERE entity_id = ? and attribute_type =? and attribute_key = ? RETURNING version;"; private static final String INSERT_OR_UPDATE = - "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts, version) " + - "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?, nextval('attribute_kv_version_seq')) " + + "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?) " + "ON CONFLICT (entity_id, attribute_type, attribute_key) " + - "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') RETURNING version;"; + "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = attribute_kv.version + 1 RETURNING version;"; @Override protected void setOnBatchUpdateValues(PreparedStatement ps, int i, List entities) throws SQLException { @@ -121,4 +121,4 @@ public class AttributeKvInsertRepository extends AbstractVersionedInsertReposito protected String getInsertOrUpdateQuery() { return INSERT_OR_UPDATE; } -} \ No newline at end of file +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index 323eec0851..0484337a08 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -30,7 +30,6 @@ import java.sql.SQLException; import java.sql.Types; import java.util.List; - @SqlTsLatestAnyDao @Repository @Transactional @@ -41,11 +40,11 @@ public class SqlLatestInsertTsRepository extends AbstractVersionedInsertReposito private Boolean updateByLatestTs; private static final String BATCH_UPDATE = - "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = nextval('ts_kv_latest_version_seq') WHERE entity_id = ? AND key = ?"; + "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = version + 1 WHERE entity_id = ? AND key = ?"; private static final String INSERT_OR_UPDATE = - "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), nextval('ts_kv_latest_version_seq')) " + - "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = nextval('ts_kv_latest_version_seq')"; + "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " + + "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = ts_kv_latest.version + 1"; private static final String BATCH_UPDATE_BY_LATEST_TS = BATCH_UPDATE + " AND ts_kv_latest.ts <= ?"; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 45deaf931d..9293636f0b 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -102,8 +102,6 @@ CREATE TABLE IF NOT EXISTS audit_log ( action_failure_details varchar(1000000) ) PARTITION BY RANGE (created_time); -CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1000; - CREATE TABLE IF NOT EXISTS attribute_kv ( entity_id uuid, attribute_type int, @@ -114,7 +112,7 @@ CREATE TABLE IF NOT EXISTS attribute_kv ( dbl_v double precision, json_v json, last_update_ts bigint, - version bigint, + version bigint default 0, CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); @@ -541,8 +539,6 @@ CREATE TABLE IF NOT EXISTS entity_view ( CONSTRAINT entity_view_external_id_unq_key UNIQUE (tenant_id, external_id) ); -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; - CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, @@ -553,7 +549,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, - version bigint, + version bigint default 0, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index e5d44cb24a..2d2381403d 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -34,8 +34,6 @@ CREATE TABLE IF NOT EXISTS key_dictionary ( CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; - CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, key int NOT NULL, @@ -45,7 +43,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest ( long_v bigint, dbl_v double precision, json_v json, - version bigint, + version bigint default 0, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-ts-latest-psql.sql b/dao/src/main/resources/sql/schema-ts-latest-psql.sql index adae100823..058104d4f2 100644 --- a/dao/src/main/resources/sql/schema-ts-latest-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-latest-psql.sql @@ -14,8 +14,6 @@ -- limitations under the License. -- -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; - CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, @@ -26,6 +24,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, - version bigint, + version bigint default 0, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); \ No newline at end of file diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index c731e239af..9c772df45b 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -23,7 +23,6 @@ DROP TABLE IF EXISTS alarm_type; DROP TABLE IF EXISTS asset; DROP TABLE IF EXISTS audit_log; DROP TABLE IF EXISTS attribute_kv; -DROP SEQUENCE IF EXISTS attribute_kv_version_seq; DROP TABLE IF EXISTS component_descriptor; DROP TABLE IF EXISTS customer; DROP TABLE IF EXISTS device; @@ -38,7 +37,6 @@ DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS ts_kv; DROP TABLE IF EXISTS ts_kv_latest; DROP TABLE IF EXISTS ts_kv_dictionary; -DROP SEQUENCE IF EXISTS ts_kv_latest_version_seq; DROP TABLE IF EXISTS user_credentials; DROP TABLE IF EXISTS widgets_bundle_widget; DROP TABLE IF EXISTS widget_type; From 118407d982cea7e2cfff7937511560cf96b8736a Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 26 Jun 2024 15:51:32 +0200 Subject: [PATCH 22/42] implemented versioned cache --- .../cache/RedisTbTransactionalCache.java | 15 +- ...VersionedCaffeineTbTransactionalCache.java | 91 ++++++++++ .../VersionedRedisTbTransactionalCache.java | 163 ++++++++++++++++++ .../cache/VersionedTbTransactionalCache.java | 33 ++++ .../server/common/data/HasVersion.java | 20 +++ .../common/data/kv/AttributeKvEntry.java | 4 +- .../common/data/kv/BaseAttributeKvEntry.java | 14 ++ .../server/common/data/util/TbPair.java | 7 + .../AbstractVersionedInsertRepository.java | 18 +- .../attributes/AttributeCaffeineCache.java | 4 +- .../dao/attributes/AttributeRedisCache.java | 33 ++-- .../server/dao/attributes/AttributesDao.java | 3 + .../attributes/CachedAttributesService.java | 73 ++++---- .../server/dao/model/BaseVersionedEntity.java | 20 +++ .../server/dao/model/ModelConstants.java | 1 + .../dao/model/sql/AttributeKvEntity.java | 6 +- .../server/dao/model/sql/VersionedEntity.java | 30 ++++ .../AttributeKvInsertRepository.java | 8 +- .../sql/attributes/AttributeKvRepository.java | 12 +- .../dao/sql/attributes/JpaAttributeDao.java | 15 +- .../sql/SqlLatestInsertTsRepository.java | 6 +- .../main/resources/sql/schema-entities.sql | 8 +- .../main/resources/sql/schema-timescale.sql | 4 +- .../resources/sql/schema-ts-latest-psql.sql | 6 +- .../resources/sql/psql/drop-all-tables.sql | 2 + 25 files changed, 492 insertions(+), 104 deletions(-) create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbTransactionalCache.java create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbTransactionalCache.java create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbTransactionalCache.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/BaseVersionedEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/VersionedEntity.java diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index abfbb398c9..543739aad4 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -29,7 +29,6 @@ import org.springframework.data.redis.core.types.Expiration; import org.springframework.data.redis.serializer.RedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; import org.thingsboard.server.common.data.FstStatsService; -import redis.clients.jedis.Connection; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.util.JedisClusterCRC16; @@ -44,7 +43,7 @@ import java.util.concurrent.TimeUnit; @Slf4j public abstract class RedisTbTransactionalCache implements TbTransactionalCache { - private static final byte[] BINARY_NULL_VALUE = RedisSerializer.java().serialize(NullValue.INSTANCE); + static final byte[] BINARY_NULL_VALUE = RedisSerializer.java().serialize(NullValue.INSTANCE); static final JedisPool MOCK_POOL = new JedisPool(); //non-null pool required for JedisConnection to trigger closing jedis connection @Autowired @@ -79,7 +78,7 @@ public abstract class RedisTbTransactionalCache get(K key) { try (var connection = connectionFactory.getConnection()) { byte[] rawKey = getRawKey(key); - byte[] rawValue = connection.get(rawKey); + byte[] rawValue = doGet(connection, rawKey); if (rawValue == null) { return null; } else if (Arrays.equals(rawValue, BINARY_NULL_VALUE)) { @@ -96,6 +95,10 @@ public abstract class RedisTbTransactionalCache(this, connection); } - private RedisConnection getConnection(byte[] rawKey) { + protected RedisConnection getConnection(byte[] rawKey) { if (!connectionFactory.isRedisClusterAware()) { return connectionFactory.getConnection(); } @@ -180,7 +183,7 @@ public abstract class RedisTbTransactionalCache implements VersionedTbTransactionalCache { + + private final CacheManager cacheManager; + private final String cacheName; + + private final Lock lock = new ReentrantLock(); + + @Override + public TbCacheValueWrapper get(K key) { + return SimpleTbCacheValueWrapper.wrap(doGet(key).getSecond()); + } + + @Override + public void put(K key, V value) { + Long version = value != null ? value.getVersion() : 0; + put(key, value, version); + } + + @Override + public void put(K key, V value, Long version) { + if (version == null) { + return; + } + lock.lock(); + try { + TbPair versionValuePair = doGet(key); + Long currentVersion = versionValuePair.getFirst(); + if (currentVersion == null || version >= currentVersion) { + cacheManager.getCache(cacheName).put(key, TbPair.of(version, value)); + } + } finally { + lock.unlock(); + } + } + + private TbPair doGet(K key) { + Cache.ValueWrapper source = cacheManager.getCache(cacheName).get(key); + return source == null ? TbPair.emptyPair() : (TbPair) source.get(); + } + + @Override + public void evict(K key) { + lock.lock(); + try { + cacheManager.getCache(cacheName).evict(key); + } finally { + lock.unlock(); + } + } + + @Override + public void evict(K key, Long version) { + if (version == null) { + return; + } + lock.lock(); + try { + put(key, null, version); + } finally { + lock.unlock(); + } + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbTransactionalCache.java new file mode 100644 index 0000000000..34b46d535a --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbTransactionalCache.java @@ -0,0 +1,163 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.NotImplementedException; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.ReturnType; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.thingsboard.server.common.data.HasVersion; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +@Slf4j +public abstract class VersionedRedisTbTransactionalCache extends RedisTbTransactionalCache implements VersionedTbTransactionalCache { + + private static final int VERSION_SIZE = 8; + private static final int VALUE_END_OFFSET = -1; + + static final byte[] SET_VERSIONED_VALUE_LUA_SCRIPT = StringRedisSerializer.UTF_8.serialize(""" + -- KEYS[1] is the key + -- ARGV[1] is the new value + -- ARGV[2] is the new version + + local key = KEYS[1] + local newValue = ARGV[1] + local newVersion = tonumber(ARGV[2]) + + -- Function to set the new value with the version + local function setNewValue() + local newValueWithVersion = struct.pack(">I8", newVersion) .. newValue:sub(9) + redis.call('SET', key, newValueWithVersion) + end + + -- Get the current version (first 8 bytes) of the current value + local currentVersionBytes = redis.call('GETRANGE', key, 0, 7) + + if currentVersionBytes and #currentVersionBytes == 8 then + -- Extract the current version from the first 8 bytes + local currentVersion = tonumber(struct.unpack(">I8", currentVersionBytes)) + + if newVersion >= currentVersion then + setNewValue() + end + else + -- If the current value is absent or the current version is not found, set the new value + setNewValue() + end + """); + static final byte[] SET_VERSIONED_VALUE_SHA = StringRedisSerializer.UTF_8.serialize("041b109dd56f6c8afb55090076e754727a5d3da0"); + + public VersionedRedisTbTransactionalCache(String cacheName, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory, TBRedisCacheConfiguration configuration, TbRedisSerializer valueSerializer) { + super(cacheName, cacheSpecsMap, connectionFactory, configuration, valueSerializer); + } + + @PostConstruct + public void init() { + try (var connection = getConnection(SET_VERSIONED_VALUE_SHA)) { + log.debug("Loading LUA with expected SHA[{}], connection [{}]", new String(SET_VERSIONED_VALUE_SHA), connection.getNativeConnection()); + String sha = connection.scriptingCommands().scriptLoad(SET_VERSIONED_VALUE_LUA_SCRIPT); + if (!Arrays.equals(SET_VERSIONED_VALUE_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { + log.error("SHA for SET_VERSIONED_VALUE_LUA_SCRIPT wrong! Expected [{}], but actual [{}], connection [{}]", new String(SET_VERSIONED_VALUE_SHA), sha, connection.getNativeConnection()); + } + } catch (Throwable t) { + log.error("Error on Redis versioned cache init", t); + } + } + + @Override + protected byte[] doGet(RedisConnection connection, byte[] rawKey) { + return connection.stringCommands().getRange(rawKey, VERSION_SIZE, VALUE_END_OFFSET); + } + + @Override + public void put(K key, V value) { + Long version = value!= null ? value.getVersion() : 0; + put(key, value, version); + } + + @Override + public void put(K key, V value, Long version) { + //TODO: use expiration + log.trace("put [{}][{}][{}]", key, value, version); + if (version == null) { + return; + } + final byte[] rawKey = getRawKey(key); + try (var connection = getConnection(rawKey)) { + byte[] rawValue = getRawValue(value); + byte[] rawVersion = StringRedisSerializer.UTF_8.serialize(String.valueOf(version)); + try { + connection.scriptingCommands().evalSha(SET_VERSIONED_VALUE_SHA, ReturnType.VALUE, 1, rawKey, rawValue, rawVersion); + } catch (InvalidDataAccessApiUsageException e) { + log.debug("loading LUA [{}]", connection.getNativeConnection()); + String sha = connection.scriptingCommands().scriptLoad(SET_VERSIONED_VALUE_LUA_SCRIPT); + if (!Arrays.equals(SET_VERSIONED_VALUE_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { + log.error("SHA for SET_VERSIONED_VALUE_LUA_SCRIPT wrong! Expected [{}], but actual [{}]", new String(SET_VERSIONED_VALUE_SHA), sha); + } + try { + connection.scriptingCommands().evalSha(SET_VERSIONED_VALUE_SHA, ReturnType.VALUE, 1, rawKey, rawValue, rawVersion); + } catch (InvalidDataAccessApiUsageException ignored) { + log.debug("Slowly executing eval instead of fast evalsha"); + connection.scriptingCommands().eval(SET_VERSIONED_VALUE_LUA_SCRIPT, ReturnType.VALUE, 1, rawKey, rawValue, rawVersion); + } + } + } + } + + @Override + public void evict(K key, Long version) { + log.trace("evict [{}][{}]", key, version); + if (version != null) { + //TODO: use evict expiration + put(key, null, version); + } + } + + @Override + public void putIfAbsent(K key, V value) { + log.trace("putIfAbsent [{}][{}]", key, value); + throw new NotImplementedException("putIfAbsent is not supported by versioned cache"); + } + + @Override + public void evict(Collection keys) { + throw new NotImplementedException("evict by many keys is not supported by versioned cache"); + } + + @Override + public void evictOrPut(K key, V value) { + throw new NotImplementedException("evictOrPut is not supported by versioned cache"); + } + + @Override + public TbCacheTransaction newTransactionForKey(K key) { + throw new NotImplementedException("newTransactionForKey is not supported by versioned cache"); + } + + @Override + public TbCacheTransaction newTransactionForKeys(List keys) { + throw new NotImplementedException("newTransactionForKeys is not supported by versioned cache"); + } + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbTransactionalCache.java new file mode 100644 index 0000000000..72f4a8c466 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbTransactionalCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import org.thingsboard.server.common.data.HasVersion; + +import java.io.Serializable; + +public interface VersionedTbTransactionalCache { + + TbCacheValueWrapper get(K key); + + void put(K key, V value); + + void put(K key, V value, Long version); + + void evict(K key); + + void evict(K key, Long version); +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java new file mode 100644 index 0000000000..fbeb8ef3b6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public interface HasVersion { + long getVersion(); +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributeKvEntry.java index 19057fb1aa..c63c953170 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributeKvEntry.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.common.data.kv; +import org.thingsboard.server.common.data.HasVersion; + /** * @author Andrew Shvayka */ -public interface AttributeKvEntry extends KvEntry { +public interface AttributeKvEntry extends KvEntry, HasVersion { long getLastUpdateTs(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index ced1486c14..6d48693511 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -29,9 +29,18 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { @Valid private final KvEntry kv; + private final Long version; + public BaseAttributeKvEntry(KvEntry kv, long lastUpdateTs) { this.kv = kv; this.lastUpdateTs = lastUpdateTs; + this.version = null; + } + + public BaseAttributeKvEntry(KvEntry kv, long lastUpdateTs, Long version) { + this.kv = kv; + this.lastUpdateTs = lastUpdateTs; + this.version = version; } public BaseAttributeKvEntry(long lastUpdateTs, KvEntry kv) { @@ -88,6 +97,11 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { return kv.getValue(); } + @Override + public long getVersion() { + return version; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java index 6e8b2e5696..315131bdad 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java @@ -21,10 +21,17 @@ import lombok.Data; @Data @AllArgsConstructor public class TbPair { + public static final TbPair EMPTY = new TbPair<>(null, null); + private S first; private T second; public static TbPair of(S first, T second) { return new TbPair<>(first, second); } + + @SuppressWarnings("unchecked") + public static TbPair emptyPair() { + return (TbPair) EMPTY; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java index cb92ca5e1f..6380957f77 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java @@ -29,9 +29,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -public abstract class AbstractVersionedInsertRepository extends AbstractInsertRepository { +import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN; - public static final String VERSION_COLUMN = "version"; +public abstract class AbstractVersionedInsertRepository extends AbstractInsertRepository { public List saveOrUpdate(List entities) { return transactionTemplate.execute(status -> { @@ -51,7 +51,7 @@ public abstract class AbstractVersionedInsertRepository extends AbstractInser for (int i = 0; i < updateResult.length; i++) { if (updateResult[i] == 0) { insertEntities.add(entities.get(i)); - seqNumbers.add(0L); + seqNumbers.add(null); toInsertIndexes.add(i); } else { seqNumbers.add((Long) seqNumbersList.get(keyHolderIndex).get(VERSION_COLUMN)); @@ -63,12 +63,14 @@ public abstract class AbstractVersionedInsertRepository extends AbstractInser return seqNumbers; } - onInsertOrUpdate(insertEntities, keyHolder); + int[] insertResult = onInsertOrUpdate(insertEntities, keyHolder); seqNumbersList = keyHolder.getKeyList(); - for (int i = 0; i < seqNumbersList.size(); i++) { - seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(VERSION_COLUMN)); + for (int i = 0; i < insertResult.length; i++) { + if (updateResult[i] != 0) { + seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(VERSION_COLUMN)); + } } return seqNumbers; @@ -89,8 +91,8 @@ public abstract class AbstractVersionedInsertRepository extends AbstractInser }, keyHolder); } - private void onInsertOrUpdate(List insertEntities, KeyHolder keyHolder) { - jdbcTemplate.batchUpdate(new SequencePreparedStatementCreator(getInsertOrUpdateQuery()), new BatchPreparedStatementSetter() { + private int[] onInsertOrUpdate(List insertEntities, KeyHolder keyHolder) { + return jdbcTemplate.batchUpdate(new SequencePreparedStatementCreator(getInsertOrUpdateQuery()), new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { setOnInsertOrUpdateValues(ps, i, insertEntities); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java index 7df55a33fb..e00ea8d11c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java @@ -18,13 +18,13 @@ package org.thingsboard.server.dao.attributes; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; -import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.cache.VersionedCaffeineTbTransactionalCache; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) @Service("AttributeCache") -public class AttributeCaffeineCache extends CaffeineTbTransactionalCache { +public class AttributeCaffeineCache extends VersionedCaffeineTbTransactionalCache { public AttributeCaffeineCache(CacheManager cacheManager) { super(cacheManager, CacheConstants.ATTRIBUTES_CACHE); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java index 7932a8b8d4..68caf62ceb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java @@ -21,9 +21,9 @@ import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; -import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.VersionedRedisTbTransactionalCache; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; @@ -38,7 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("AttributeCache") -public class AttributeRedisCache extends RedisTbTransactionalCache { +public class AttributeRedisCache extends VersionedRedisTbTransactionalCache { public AttributeRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { super(CacheConstants.ATTRIBUTES_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>() { @@ -82,26 +82,15 @@ public class AttributeRedisCache extends RedisTbTransactionalCache new BooleanDataEntry(key.getKey(), hasValue ? proto.getBoolV() : null); + case LONG_V -> new LongDataEntry(key.getKey(), hasValue ? proto.getLongV() : null); + case DOUBLE_V -> new DoubleDataEntry(key.getKey(), hasValue ? proto.getDoubleV() : null); + case STRING_V -> new StringDataEntry(key.getKey(), hasValue ? proto.getStringV() : null); + case JSON_V -> new JsonDataEntry(key.getKey(), hasValue ? proto.getJsonV() : null); + default -> + throw new InvalidProtocolBufferException("Unrecognized type: " + proto.getType() + " !"); + }; return new BaseAttributeKvEntry(proto.getLastUpdateTs(), entry); } catch (InvalidProtocolBufferException e) { throw new SerializationException(e.getMessage()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 34f67fb5ff..1805f72a8f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.util.TbPair; import java.util.Collection; import java.util.List; @@ -42,6 +43,8 @@ public interface AttributesDao { List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); + List>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); + List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 375b56b1f6..fb5c745a11 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -27,13 +27,14 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; -import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.cache.VersionedTbTransactionalCache; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; @@ -67,7 +68,7 @@ public class CachedAttributesService implements AttributesService { private final CacheExecutorService cacheExecutorService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; - private final TbTransactionalCache cache; + private final VersionedTbTransactionalCache cache; private ListeningExecutorService cacheExecutor; @Value("${cache.type:caffeine}") @@ -79,7 +80,7 @@ public class CachedAttributesService implements AttributesService { JpaExecutorService jpaExecutorService, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, - TbTransactionalCache cache) { + VersionedTbTransactionalCache cache) { this.attributesDao = attributesDao; this.jpaExecutorService = jpaExecutorService; this.cacheExecutorService = cacheExecutorService; @@ -122,17 +123,9 @@ public class CachedAttributesService implements AttributesService { return Optional.ofNullable(cachedAttributeKvEntry); } else { missCounter.increment(); - var cacheTransaction = cache.newTransactionForKey(attributeCacheKey); - try { - Optional result = attributesDao.find(tenantId, entityId, scope, attributeKey); - cacheTransaction.putIfAbsent(attributeCacheKey, result.orElse(null)); - cacheTransaction.commit(); - return result; - } catch (Throwable e) { - cacheTransaction.rollback(); - log.debug("Could not find attribute from cache: [{}] [{}] [{}]", entityId, scope, attributeKey, e); - throw e; - } + Optional result = attributesDao.find(tenantId, entityId, scope, attributeKey); + cache.put(attributeCacheKey, result.orElse(null)); + return result; } }); } @@ -163,28 +156,19 @@ public class CachedAttributesService implements AttributesService { // DB call should run in DB executor, not in cache-related executor return jpaExecutorService.submit(() -> { - var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); - try { - log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); - List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); - for (AttributeKvEntry foundInDbAttribute : result) { - AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); - cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); - notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); - } - for (String key : notFoundAttributeKeys) { - cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); - } - List mergedAttributes = new ArrayList<>(cachedAttributes); - mergedAttributes.addAll(result); - cacheTransaction.commit(); - log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); - return mergedAttributes; - } catch (Throwable e) { - cacheTransaction.rollback(); - log.debug("Could not find attributes from cache: [{}] [{}] [{}]", entityId, scope, notFoundAttributeKeys, e); - throw e; + log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); + List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); + for (AttributeKvEntry foundInDbAttribute : result) { + put(entityId, scope, foundInDbAttribute, foundInDbAttribute.getVersion()); + notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); } + for (String key : notFoundAttributeKeys) { + cache.put(new AttributeCacheKey(scope, entityId, key), null); + } + List mergedAttributes = new ArrayList<>(cachedAttributes); + mergedAttributes.addAll(result); + log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); + return mergedAttributes; }); }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor @@ -235,7 +219,7 @@ public class CachedAttributesService implements AttributesService { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> evict(entityId, scope, attribute, version), cacheExecutor); + return Futures.transform(future, version -> put(entityId, scope, attribute, version), cacheExecutor); } @Override @@ -251,17 +235,17 @@ public class CachedAttributesService implements AttributesService { List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - futures.add(Futures.transform(future, version -> evict(entityId, scope, attribute, version), cacheExecutor)); + futures.add(Futures.transform(future, version -> put(entityId, scope, attribute, version), cacheExecutor)); } return Futures.allAsList(futures); } - private Long evict(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute, Long version) { + private Long put(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute, Long version) { String key = attribute.getKey(); - log.trace("[{}][{}][{}] Before cache evict: {}", entityId, scope, key, attribute); - cache.evictOrPut(new AttributeCacheKey(scope, entityId, key), attribute); - log.trace("[{}][{}][{}] after cache evict.", entityId, scope, key); + log.trace("[{}][{}][{}] Before cache put: {}", entityId, scope, key, attribute); + cache.put(new AttributeCacheKey(scope, entityId, key), attribute, version); + log.trace("[{}][{}][{}] after cache put.", entityId, scope, key); return version; } @@ -273,9 +257,10 @@ public class CachedAttributesService implements AttributesService { @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - List> futures = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); - return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, key -> { - cache.evict(new AttributeCacheKey(scope, entityId, key)); + List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); + return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, keyVersionPair -> { + String key = keyVersionPair.getFirst(); + cache.evict(new AttributeCacheKey(scope, entityId, key), keyVersionPair.getSecond()); return key; }, cacheExecutor)).collect(Collectors.toList())); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/BaseVersionedEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/BaseVersionedEntity.java new file mode 100644 index 0000000000..6b98348de9 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/BaseVersionedEntity.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model; + +public interface BaseVersionedEntity { + long getVersion(); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index fb77ad6987..643c099829 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -56,6 +56,7 @@ public class ModelConstants { public static final String ATTRIBUTE_TYPE_COLUMN = "attribute_type"; public static final String ATTRIBUTE_KEY_COLUMN = "attribute_key"; public static final String LAST_UPDATE_TS_COLUMN = "last_update_ts"; + public static final String VERSION_COLUMN = "version"; /** * User constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java index 03aa0f98ad..d7bd23db4d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; @@ -41,9 +42,10 @@ import static org.thingsboard.server.dao.model.ModelConstants.LONG_VALUE_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUMN; @Data +@EqualsAndHashCode(callSuper = true) @Entity @Table(name = "attribute_kv") -public class AttributeKvEntity implements ToData, Serializable { +public class AttributeKvEntity extends VersionedEntity implements ToData, Serializable { @EmbeddedId private AttributeKvCompositeKey id; @@ -84,6 +86,6 @@ public class AttributeKvEntity implements ToData, Serializable kvEntry = new JsonDataEntry(strKey, jsonValue); } - return new BaseAttributeKvEntry(kvEntry, lastUpdateTs); + return new BaseAttributeKvEntry(kvEntry, lastUpdateTs, version); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/VersionedEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/VersionedEntity.java new file mode 100644 index 0000000000..2dfee6466a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/VersionedEntity.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.MappedSuperclass; +import lombok.Data; + +import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN; + +@Data +@MappedSuperclass +public abstract class VersionedEntity { + + @Column(name = VERSION_COLUMN) + protected Long version; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index c2b9d53d47..7050e61144 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -31,14 +31,14 @@ import java.util.List; @SqlDao public class AttributeKvInsertRepository extends AbstractVersionedInsertRepository { - private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = version + 1 " + + private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') " + "WHERE entity_id = ? and attribute_type =? and attribute_key = ? RETURNING version;"; private static final String INSERT_OR_UPDATE = - "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts) " + - "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?) " + + "INSERT INTO attribute_kv (entity_id, attribute_type, attribute_key, str_v, long_v, dbl_v, bool_v, json_v, last_update_ts, version) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), ?, nextval('attribute_kv_version_seq')) " + "ON CONFLICT (entity_id, attribute_type, attribute_key) " + - "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = attribute_kv.version + 1 RETURNING version;"; + "DO UPDATE SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, json_v = cast(? AS json), last_update_ts = ?, version = nextval('attribute_kv_version_seq') RETURNING version;"; @Override protected void setOnBatchUpdateValues(PreparedStatement ps, int i, List entities) throws SQLException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index ddd6c3c881..beea5f2f2f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -30,15 +30,15 @@ public interface AttributeKvRepository extends JpaRepository findAllEntityIdAndAttributeType(@Param("entityId") UUID entityId, - @Param("attributeType") int attributeType); + List findAllByEntityIdAndAttributeType(@Param("entityId") UUID entityId, + @Param("attributeType") int attributeType); @Transactional @Modifying - @Query("DELETE FROM AttributeKvEntity a WHERE a.id.entityId = :entityId " + - "AND a.id.attributeType = :attributeType " + - "AND a.id.attributeKey = :attributeKey") - void delete(@Param("entityId") UUID entityId, + @Query(value = "DELETE FROM attribute_kv WHERE entity_id = :entityId " + + "AND attribute_type = :attributeType " + + "AND attribute_key = :attributeKey RETURNING version", nativeQuery = true) + Long delete(@Param("entityId") UUID entityId, @Param("attributeType") int attributeType, @Param("attributeKey") int attributeKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 05cdfce221..de4487381b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; @@ -144,7 +145,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Override public List findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope) { - List attributes = attributeKvRepository.findAllEntityIdAndAttributeType( + List attributes = attributeKvRepository.findAllByEntityIdAndAttributeType( entityId.getId(), attributeScope.getId()); attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(keyDictionaryDao.getKey(attributeKvEntity.getId().getAttributeKey()))); @@ -205,6 +206,18 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return futuresList; } + @Override + public List>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys) { + List>> futuresList = new ArrayList<>(keys.size()); + for (String key : keys) { + futuresList.add(service.submit(() -> { + Long version = attributeKvRepository.delete(entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(key)); + return TbPair.of(key, version); + })); + } + return futuresList; + } + @Transactional @Override public List> removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java index 0484337a08..0a3ffad09f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java @@ -40,11 +40,11 @@ public class SqlLatestInsertTsRepository extends AbstractVersionedInsertReposito private Boolean updateByLatestTs; private static final String BATCH_UPDATE = - "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = version + 1 WHERE entity_id = ? AND key = ?"; + "UPDATE ts_kv_latest SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = nextval('ts_kv_latest_version_seq') WHERE entity_id = ? AND key = ?"; private static final String INSERT_OR_UPDATE = - "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json)) " + - "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = ts_kv_latest.version + 1"; + "INSERT INTO ts_kv_latest (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version) VALUES(?, ?, ?, ?, ?, ?, ?, cast(? AS json), nextval('ts_kv_latest_version_seq')) " + + "ON CONFLICT (entity_id, key) DO UPDATE SET ts = ?, bool_v = ?, str_v = ?, long_v = ?, dbl_v = ?, json_v = cast(? AS json), version = nextval('ts_kv_latest_version_seq')"; private static final String BATCH_UPDATE_BY_LATEST_TS = BATCH_UPDATE + " AND ts_kv_latest.ts <= ?"; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 9293636f0b..c2e3a2b108 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -102,6 +102,8 @@ CREATE TABLE IF NOT EXISTS audit_log ( action_failure_details varchar(1000000) ) PARTITION BY RANGE (created_time); +CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1000; + CREATE TABLE IF NOT EXISTS attribute_kv ( entity_id uuid, attribute_type int, @@ -112,7 +114,7 @@ CREATE TABLE IF NOT EXISTS attribute_kv ( dbl_v double precision, json_v json, last_update_ts bigint, - version bigint default 0, + version bigint default 1, CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); @@ -539,6 +541,8 @@ CREATE TABLE IF NOT EXISTS entity_view ( CONSTRAINT entity_view_external_id_unq_key UNIQUE (tenant_id, external_id) ); +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; + CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, @@ -549,7 +553,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, - version bigint default 0, + version bigint default 1, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 2d2381403d..7c2a99fba3 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -34,6 +34,8 @@ CREATE TABLE IF NOT EXISTS key_dictionary ( CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; + CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, key int NOT NULL, @@ -43,7 +45,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest ( long_v bigint, dbl_v double precision, json_v json, - version bigint default 0, + version bigint default 1, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-ts-latest-psql.sql b/dao/src/main/resources/sql/schema-ts-latest-psql.sql index 058104d4f2..d50571c091 100644 --- a/dao/src/main/resources/sql/schema-ts-latest-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-latest-psql.sql @@ -14,6 +14,8 @@ -- limitations under the License. -- +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; + CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, @@ -24,6 +26,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, - version bigint default 0, + version bigint default 1, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); \ No newline at end of file +); diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index 9c772df45b..e1c35d7a21 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -23,6 +23,7 @@ DROP TABLE IF EXISTS alarm_type; DROP TABLE IF EXISTS asset; DROP TABLE IF EXISTS audit_log; DROP TABLE IF EXISTS attribute_kv; +DROP SEQUENCE IF EXISTS attribute_kv_version_seq; DROP TABLE IF EXISTS component_descriptor; DROP TABLE IF EXISTS customer; DROP TABLE IF EXISTS device; @@ -36,6 +37,7 @@ DROP TABLE IF EXISTS relation; DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS ts_kv; DROP TABLE IF EXISTS ts_kv_latest; +DROP SEQUENCE IF EXISTS ts_kv_latest_version_seq; DROP TABLE IF EXISTS ts_kv_dictionary; DROP TABLE IF EXISTS user_credentials; DROP TABLE IF EXISTS widgets_bundle_widget; From 6162060e88a04e8883f54dec8f1d78e65d47fb74 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 1 Jul 2024 12:24:27 +0200 Subject: [PATCH 23/42] fixed insert result and used expiration --- .../cache/RedisTbTransactionalCache.java | 4 +- ...che.java => VersionedCaffeineTbCache.java} | 4 +- ...lCache.java => VersionedRedisTbCache.java} | 39 +++++++++---------- ...tionalCache.java => VersionedTbCache.java} | 2 +- .../AbstractVersionedInsertRepository.java | 2 +- .../attributes/AttributeCaffeineCache.java | 4 +- .../dao/attributes/AttributeRedisCache.java | 4 +- .../attributes/CachedAttributesService.java | 6 +-- .../sql/attributes/AttributeKvRepository.java | 3 +- .../main/resources/sql/schema-entities.sql | 4 +- .../main/resources/sql/schema-timescale.sql | 2 +- .../resources/sql/schema-ts-latest-psql.sql | 2 +- .../attributes/BaseAttributesServiceTest.java | 22 +---------- 13 files changed, 39 insertions(+), 59 deletions(-) rename common/cache/src/main/java/org/thingsboard/server/cache/{VersionedCaffeineTbTransactionalCache.java => VersionedCaffeineTbCache.java} (91%) rename common/cache/src/main/java/org/thingsboard/server/cache/{VersionedRedisTbTransactionalCache.java => VersionedRedisTbCache.java} (82%) rename common/cache/src/main/java/org/thingsboard/server/cache/{VersionedTbTransactionalCache.java => VersionedTbCache.java} (89%) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index 543739aad4..e33e13caa1 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -54,8 +54,8 @@ public abstract class RedisTbTransactionalCache keySerializer = StringRedisSerializer.UTF_8; private final TbRedisSerializer valueSerializer; - private final Expiration evictExpiration; - private final Expiration cacheTtl; + protected final Expiration evictExpiration; + protected final Expiration cacheTtl; public RedisTbTransactionalCache(String cacheName, CacheSpecsMap cacheSpecsMap, diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java similarity index 91% rename from common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbTransactionalCache.java rename to common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java index f2120cf38a..e084668b90 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java @@ -26,7 +26,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @RequiredArgsConstructor -public abstract class VersionedCaffeineTbTransactionalCache implements VersionedTbTransactionalCache { +public abstract class VersionedCaffeineTbCache implements VersionedTbCache { private final CacheManager cacheManager; private final String cacheName; @@ -53,7 +53,7 @@ public abstract class VersionedCaffeineTbTransactionalCache versionValuePair = doGet(key); Long currentVersion = versionValuePair.getFirst(); - if (currentVersion == null || version >= currentVersion) { + if (currentVersion == null || version > currentVersion) { cacheManager.getCache(cacheName).put(key, TbPair.of(version, value)); } } finally { diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java similarity index 82% rename from common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbTransactionalCache.java rename to common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java index 34b46d535a..5e9ba187c9 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java @@ -22,6 +22,7 @@ import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.connection.ReturnType; +import org.springframework.data.redis.core.types.Expiration; import org.springframework.data.redis.serializer.StringRedisSerializer; import org.thingsboard.server.common.data.HasVersion; @@ -31,34 +32,29 @@ import java.util.Collection; import java.util.List; @Slf4j -public abstract class VersionedRedisTbTransactionalCache extends RedisTbTransactionalCache implements VersionedTbTransactionalCache { +public abstract class VersionedRedisTbCache extends RedisTbTransactionalCache implements VersionedTbCache { private static final int VERSION_SIZE = 8; private static final int VALUE_END_OFFSET = -1; static final byte[] SET_VERSIONED_VALUE_LUA_SCRIPT = StringRedisSerializer.UTF_8.serialize(""" - -- KEYS[1] is the key - -- ARGV[1] is the new value - -- ARGV[2] is the new version - local key = KEYS[1] local newValue = ARGV[1] local newVersion = tonumber(ARGV[2]) + local expiration = tonumber(ARGV[3]) - -- Function to set the new value with the version local function setNewValue() local newValueWithVersion = struct.pack(">I8", newVersion) .. newValue:sub(9) - redis.call('SET', key, newValueWithVersion) + redis.call('SET', key, newValueWithVersion, 'EX', expiration) end -- Get the current version (first 8 bytes) of the current value local currentVersionBytes = redis.call('GETRANGE', key, 0, 7) if currentVersionBytes and #currentVersionBytes == 8 then - -- Extract the current version from the first 8 bytes local currentVersion = tonumber(struct.unpack(">I8", currentVersionBytes)) - if newVersion >= currentVersion then + if newVersion > currentVersion then setNewValue() end else @@ -66,9 +62,9 @@ public abstract class VersionedRedisTbTransactionalCache valueSerializer) { + public VersionedRedisTbCache(String cacheName, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory, TBRedisCacheConfiguration configuration, TbRedisSerializer valueSerializer) { super(cacheName, cacheSpecsMap, connectionFactory, configuration, valueSerializer); } @@ -92,23 +88,27 @@ public abstract class VersionedRedisTbTransactionalCache { +public interface VersionedTbCache { TbCacheValueWrapper get(K key); diff --git a/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java index 6380957f77..43ed41e52a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/AbstractVersionedInsertRepository.java @@ -68,7 +68,7 @@ public abstract class AbstractVersionedInsertRepository extends AbstractInser seqNumbersList = keyHolder.getKeyList(); for (int i = 0; i < insertResult.length; i++) { - if (updateResult[i] != 0) { + if (insertResult[i] != 0) { seqNumbers.set(toInsertIndexes.get(i), (Long) seqNumbersList.get(i).get(VERSION_COLUMN)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java index e00ea8d11c..140cfbc03e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java @@ -18,13 +18,13 @@ package org.thingsboard.server.dao.attributes; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; -import org.thingsboard.server.cache.VersionedCaffeineTbTransactionalCache; +import org.thingsboard.server.cache.VersionedCaffeineTbCache; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) @Service("AttributeCache") -public class AttributeCaffeineCache extends VersionedCaffeineTbTransactionalCache { +public class AttributeCaffeineCache extends VersionedCaffeineTbCache { public AttributeCaffeineCache(CacheManager cacheManager) { super(cacheManager, CacheConstants.ATTRIBUTES_CACHE); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java index 68caf62ceb..a6f21f9c3a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java @@ -23,7 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.cache.TbRedisSerializer; -import org.thingsboard.server.cache.VersionedRedisTbTransactionalCache; +import org.thingsboard.server.cache.VersionedRedisTbCache; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; @@ -38,7 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("AttributeCache") -public class AttributeRedisCache extends VersionedRedisTbTransactionalCache { +public class AttributeRedisCache extends VersionedRedisTbCache { public AttributeRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { super(CacheConstants.ATTRIBUTES_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index fb5c745a11..d31ff9db79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -27,7 +27,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; -import org.thingsboard.server.cache.VersionedTbTransactionalCache; +import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -68,7 +68,7 @@ public class CachedAttributesService implements AttributesService { private final CacheExecutorService cacheExecutorService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; - private final VersionedTbTransactionalCache cache; + private final VersionedTbCache cache; private ListeningExecutorService cacheExecutor; @Value("${cache.type:caffeine}") @@ -80,7 +80,7 @@ public class CachedAttributesService implements AttributesService { JpaExecutorService jpaExecutorService, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, - VersionedTbTransactionalCache cache) { + VersionedTbCache cache) { this.attributesDao = attributesDao; this.jpaExecutorService = jpaExecutorService; this.cacheExecutorService = cacheExecutorService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index beea5f2f2f..8367431946 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -37,7 +37,7 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByEntityIdsAndAttributeType(@Param("entityIds") List entityIds, @Param("attributeType") int attributeType); } - diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index c2e3a2b108..9e6ab22248 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -114,7 +114,7 @@ CREATE TABLE IF NOT EXISTS attribute_kv ( dbl_v double precision, json_v json, last_update_ts bigint, - version bigint default 1, + version bigint default 0, CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_id, attribute_type, attribute_key) ); @@ -553,7 +553,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, - version bigint default 1, + version bigint default 0, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 7c2a99fba3..1d5b67a95e 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest ( long_v bigint, dbl_v double precision, json_v json, - version bigint default 1, + version bigint default 0, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/main/resources/sql/schema-ts-latest-psql.sql b/dao/src/main/resources/sql/schema-ts-latest-psql.sql index d50571c091..4892e40176 100644 --- a/dao/src/main/resources/sql/schema-ts-latest-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-latest-psql.sql @@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest long_v bigint, dbl_v double precision, json_v json, - version bigint default 1, + version bigint default 0, CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index 98a8cf9d47..ad2470a9b1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -26,7 +26,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -60,7 +60,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { private static final String NEW_VALUE = "NEW VALUE"; @Autowired - private TbTransactionalCache cache; + private VersionedTbCache cache; @Autowired private AttributesService attributesService; @@ -132,24 +132,6 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { Assert.assertTrue(result.isEmpty()); } - @Test - public void testConcurrentTransaction() throws Exception { - var tenantId = new TenantId(UUID.randomUUID()); - var deviceId = new DeviceId(UUID.randomUUID()); - var scope = AttributeScope.SERVER_SCOPE; - var key = "TEST"; - - var attrKey = new AttributeCacheKey(scope, deviceId, "TEST"); - var oldValue = new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, OLD_VALUE)); - var newValue = new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, NEW_VALUE)); - - var trx = cache.newTransactionForKey(attrKey); - cache.putIfAbsent(attrKey, newValue); - trx.putIfAbsent(attrKey, oldValue); - Assert.assertFalse(trx.commit()); - Assert.assertEquals(NEW_VALUE, getAttributeValue(tenantId, deviceId, scope, key)); - } - @Test public void testConcurrentFetchAndUpdate() throws Exception { var tenantId = new TenantId(UUID.randomUUID()); From 7fa752854641e7aca149a6d863ffc6bfec29f8a7 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 2 Jul 2024 12:38:07 +0200 Subject: [PATCH 24/42] fixed delete attributes query --- .../server/dao/sql/attributes/AttributeKvRepository.java | 8 ++++---- .../server/dao/sql/attributes/JpaAttributeDao.java | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index 8367431946..90f49c57fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -35,10 +35,10 @@ public interface AttributeKvRepository extends JpaRepository>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys) { List>> futuresList = new ArrayList<>(keys.size()); for (String key : keys) { futuresList.add(service.submit(() -> { - Long version = attributeKvRepository.delete(entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(key)); + Long version = jdbcTemplate.query("DELETE FROM attribute_kv WHERE entity_id = ? AND attribute_type = ? " + + "AND attribute_key = ? RETURNING nextval('attribute_kv_version_seq')", + rs -> rs.next() ? rs.getLong(1) : null, entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(key)); return TbPair.of(key, version); })); } From 0b51baf38de1439224ecab046e8de938053e62de Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 3 Jul 2024 16:03:19 +0200 Subject: [PATCH 25/42] LUA script improvements and used versioned cache for latest ts --- .../cache/RedisTbTransactionalCache.java | 2 +- .../server/cache/VersionedRedisTbCache.java | 16 ++- .../server/cache}/TsLatestRedisCacheTest.java | 6 +- .../server/common/data/HasVersion.java | 2 +- .../common/data/kv/BaseAttributeKvEntry.java | 2 +- .../server/common/data/kv/BasicTsKvEntry.java | 13 ++ .../server/common/data/kv/TsKvEntry.java | 3 +- .../data/kv/TsKvLatestRemovingResult.java | 11 +- .../dao/model/sql/AbstractTsKvEntity.java | 7 +- .../model/sqlts/latest/TsKvLatestEntity.java | 11 +- ...paAbstractDaoListeningExecutorService.java | 4 + .../dao/sql/attributes/JpaAttributeDao.java | 5 +- .../CachedRedisSqlTimeseriesLatestDao.java | 51 +++---- .../dao/sqlts/SqlTimeseriesLatestDao.java | 12 +- .../latest/SearchTsKvLatestRepository.java | 2 +- .../CassandraBaseTimeseriesLatestDao.java | 2 +- .../dao/timeseries/TsLatestRedisCache.java | 127 +++--------------- .../attributes/BaseAttributesServiceTest.java | 3 - 18 files changed, 106 insertions(+), 173 deletions(-) rename {dao/src/test/java/org/thingsboard/server/dao/timeseries => common/cache/src/test/java/org/thingsboard/server/cache}/TsLatestRedisCacheTest.java (83%) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index dd371d96b6..9e51795be6 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -80,7 +80,7 @@ public abstract class RedisTbTransactionalCacheI8", newVersion) .. newValue:sub(9) + local newValueWithVersion = struct.pack(">I8", newVersion) .. newValue redis.call('SET', key, newValueWithVersion, 'EX', expiration) end + + local function bytes_to_number(bytes) + local n = 0 + for i = 1, 8 do + n = n * 256 + string.byte(bytes, i) + end + return n + end -- Get the current version (first 8 bytes) of the current value local currentVersionBytes = redis.call('GETRANGE', key, 0, 7) if currentVersionBytes and #currentVersionBytes == 8 then - local currentVersion = tonumber(struct.unpack(">I8", currentVersionBytes)) + local currentVersion = bytes_to_number(currentVersionBytes) - if newVersion > currentVersion then + if newVersion > currentVersion or newVersion == 1 and currentVersion > 1 then setNewValue() end else @@ -62,7 +70,7 @@ public abstract class VersionedRedisTbCache valueSerializer) { super(cacheName, cacheSpecsMap, connectionFactory, configuration, valueSerializer); diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java b/common/cache/src/test/java/org/thingsboard/server/cache/TsLatestRedisCacheTest.java similarity index 83% rename from dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java rename to common/cache/src/test/java/org/thingsboard/server/cache/TsLatestRedisCacheTest.java index c463b4630f..d0f3042c61 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java +++ b/common/cache/src/test/java/org/thingsboard/server/cache/TsLatestRedisCacheTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.timeseries; +package org.thingsboard.server.cache; import lombok.SneakyThrows; import org.junit.jupiter.api.Test; @@ -22,11 +22,11 @@ import java.security.MessageDigest; import static org.assertj.core.api.Assertions.assertThat; -class TsLatestRedisCacheTest { +class VersionedRedisTbCacheTest { @Test void testUpsertTsLatestLUAScriptHash() { - assertThat(getSHA1(TsLatestRedisCache.UPSERT_TS_LATEST_LUA_SCRIPT)).isEqualTo(new String(TsLatestRedisCache.UPSERT_TS_LATEST_SHA)); + assertThat(getSHA1(VersionedRedisTbCache.SET_VERSIONED_VALUE_LUA_SCRIPT)).isEqualTo(new String(VersionedRedisTbCache.SET_VERSIONED_VALUE_SHA)); } @SneakyThrows diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java index fbeb8ef3b6..b9af6f93c9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/HasVersion.java @@ -16,5 +16,5 @@ package org.thingsboard.server.common.data; public interface HasVersion { - long getVersion(); + Long getVersion(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java index 6d48693511..001e061aa9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseAttributeKvEntry.java @@ -98,7 +98,7 @@ public class BaseAttributeKvEntry implements AttributeKvEntry { } @Override - public long getVersion() { + public Long getVersion() { return version; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java index 396d1df5ba..ab39498ae0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -25,9 +25,18 @@ public class BasicTsKvEntry implements TsKvEntry { @Valid private final KvEntry kv; + private final Long version; + public BasicTsKvEntry(long ts, KvEntry kv) { this.ts = ts; this.kv = kv; + this.version = null; + } + + public BasicTsKvEntry(long ts, KvEntry kv, Long version) { + this.ts = ts; + this.kv = kv; + this.version = version; } @Override @@ -118,4 +127,8 @@ public class BasicTsKvEntry implements TsKvEntry { return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT); } + @Override + public Long getVersion() { + return version; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index c65f48e550..4b8887e893 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.kv; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.query.TsValue; /** @@ -24,7 +25,7 @@ import org.thingsboard.server.common.data.query.TsValue; * @author ashvayka * */ -public interface TsKvEntry extends KvEntry { +public interface TsKvEntry extends KvEntry, HasVersion { long getTs(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvLatestRemovingResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvLatestRemovingResult.java index acdb745ad8..dae91bdd81 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvLatestRemovingResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvLatestRemovingResult.java @@ -22,15 +22,22 @@ public class TsKvLatestRemovingResult { private String key; private TsKvEntry data; private boolean removed; + private Long version; - public TsKvLatestRemovingResult(TsKvEntry data) { + public TsKvLatestRemovingResult(String key, boolean removed) { + this(key, removed, null); + } + + public TsKvLatestRemovingResult(TsKvEntry data, Long version) { this.key = data.getKey(); this.data = data; this.removed = true; + this.version = version; } - public TsKvLatestRemovingResult(String key, boolean removed) { + public TsKvLatestRemovingResult(String key, boolean removed, Long version) { this.key = key; this.removed = removed; + this.version = version; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java index a0a4664829..ae4f47a37a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTsKvEntity.java @@ -119,10 +119,13 @@ public abstract class AbstractTsKvEntity implements ToData { } if (aggValuesCount == null) { - return new BasicTsKvEntry(ts, kvEntry); + return new BasicTsKvEntry(ts, kvEntry, getVersion()); } else { return new AggTsKvEntry(ts, kvEntry, aggValuesCount); } } -} \ No newline at end of file + public Long getVersion() { + return null; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java index 0d34bb5195..23fe580fa3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.model.sqlts.latest; +import jakarta.persistence.Column; import jakarta.persistence.ColumnResult; import jakarta.persistence.ConstructorResult; import jakarta.persistence.Entity; @@ -30,6 +31,8 @@ import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; import java.util.UUID; +import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN; + @Data @Entity @Table(name = "ts_kv_latest") @@ -50,7 +53,7 @@ import java.util.UUID; @ColumnResult(name = "doubleValue", type = Double.class), @ColumnResult(name = "jsonValue", type = String.class), @ColumnResult(name = "ts", type = Long.class), - + @ColumnResult(name = "version", type = Long.class) } ), }) @@ -65,6 +68,9 @@ import java.util.UUID; }) public final class TsKvLatestEntity extends AbstractTsKvEntity { + @Column(name = VERSION_COLUMN) + private Long version; + @Override public boolean isNotEmpty() { return strValue != null || longValue != null || doubleValue != null || booleanValue != null || jsonValue != null; @@ -73,7 +79,7 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity { public TsKvLatestEntity() { } - public TsKvLatestEntity(UUID entityId, Integer key, String strKey, String strValue, Boolean boolValue, Long longValue, Double doubleValue, String jsonValue, Long ts) { + public TsKvLatestEntity(UUID entityId, Integer key, String strKey, String strValue, Boolean boolValue, Long longValue, Double doubleValue, String jsonValue, Long ts, Long version) { this.entityId = entityId; this.key = key; this.ts = ts; @@ -83,5 +89,6 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity { this.booleanValue = boolValue; this.jsonValue = jsonValue; this.strKey = strKey; + this.version = version; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java index b85e1bc724..3ac31bf831 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDaoListeningExecutorService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.support.TransactionTemplate; import javax.sql.DataSource; import java.sql.SQLException; @@ -36,6 +37,9 @@ public abstract class JpaAbstractDaoListeningExecutorService { @Autowired protected JdbcTemplate jdbcTemplate; + @Autowired + protected TransactionTemplate transactionTemplate; + protected void printWarnings(Statement statement) throws SQLException { SQLWarning warnings = statement.getWarnings(); if (warnings != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 172940d21d..a0e1f1447d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -206,15 +206,14 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return futuresList; } - @Transactional @Override public List>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys) { List>> futuresList = new ArrayList<>(keys.size()); for (String key : keys) { futuresList.add(service.submit(() -> { - Long version = jdbcTemplate.query("DELETE FROM attribute_kv WHERE entity_id = ? AND attribute_type = ? " + + Long version = transactionTemplate.execute(status -> jdbcTemplate.query("DELETE FROM attribute_kv WHERE entity_id = ? AND attribute_type = ? " + "AND attribute_key = ? RETURNING nextval('attribute_kv_version_seq')", - rs -> rs.next() ? rs.getLong(1) : null, entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(key)); + rs -> rs.next() ? rs.getLong(1) : null, entityId.getId(), attributeScope.getId(), keyDictionaryDao.getOrSaveKeyId(key))); return TbPair.of(key, version); })); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index 32d0ee91c9..42dc149342 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -25,7 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; import org.thingsboard.server.cache.TbCacheValueWrapper; -import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -52,7 +52,7 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries final CacheExecutorService cacheExecutorService; final SqlTimeseriesLatestDao sqlDao; final StatsFactory statsFactory; - final TbTransactionalCache cache; + final VersionedTbCache cache; DefaultCounter hitCounter; DefaultCounter missCounter; @@ -64,17 +64,17 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries } @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - ListenableFuture future = sqlDao.saveLatest(tenantId, entityId, tsKvEntry); + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + ListenableFuture future = sqlDao.saveLatest(tenantId, entityId, tsKvEntry); future = Futures.transform(future, x -> { - cache.put(new TsLatestCacheKey(entityId, tsKvEntry.getKey()), tsKvEntry); + cache.put(new TsLatestCacheKey(entityId, tsKvEntry.getKey()), tsKvEntry, x); return x; }, cacheExecutorService); if (log.isTraceEnabled()) { Futures.addCallback(future, new FutureCallback<>() { @Override - public void onSuccess(Void result) { + public void onSuccess(Long result) { log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry); } @@ -91,7 +91,15 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture future = sqlDao.removeLatest(tenantId, entityId, query); future = Futures.transform(future, x -> { - cache.evict(new TsLatestCacheKey(entityId, query.getKey())); + if (x.isRemoved()) { + TsLatestCacheKey key = new TsLatestCacheKey(entityId, query.getKey()); + Long version = x.getVersion(); + if (x.getData() != null) { + cache.put(key, x.getData(), version); + } else { + cache.evict(key, version); + } + } return x; }, cacheExecutorService); @@ -133,32 +141,11 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return Futures.immediateFuture(Optional.ofNullable(tsKvEntry)); } log.debug("findLatest cache miss [{}][{}]", entityId, key); - ListenableFuture> daoFuture = sqlDao.findLatestOpt(tenantId,entityId, key); - - return Futures.transformAsync(daoFuture, (daoValue) -> { - - if (daoValue.isEmpty()) { - //TODO implement the cache logic if no latest found in TS DAO. Currently we are always getting from DB to stay on the safe side - return Futures.immediateFuture(daoValue); - } - ListenableFuture> cachePutFuture = cacheExecutorService.submit(() -> { - cache.put(new TsLatestCacheKey(entityId, key), daoValue.get()); - return daoValue; - }); - - Futures.addCallback(cachePutFuture, new FutureCallback<>() { - @Override - public void onSuccess(Optional result) { - log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, key, result); - } - - @Override - public void onFailure(Throwable t) { - log.info("saveLatest onFailure [{}][{}][{}]", entityId, key, daoValue, t); - } + ListenableFuture> daoFuture = sqlDao.findLatestOpt(tenantId, entityId, key); - }, MoreExecutors.directExecutor()); - return cachePutFuture; + return Futures.transform(daoFuture, daoValue -> { + cache.put(cacheKey, daoValue.orElse(null)); + return daoValue; }, MoreExecutors.directExecutor()); }, MoreExecutors.directExecutor()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index edb0e535d1..44859c26e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -190,7 +190,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return Futures.transformAsync(future, entryList -> { if (entryList.size() == 1) { TsKvEntry entry = entryList.get(0); - return Futures.transform(getSaveLatestFuture(entityId, entry), v -> new TsKvLatestRemovingResult(entry), MoreExecutors.directExecutor()); + return Futures.transform(getSaveLatestFuture(entityId, entry), v -> new TsKvLatestRemovingResult(entry, v), MoreExecutors.directExecutor()); } else { log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); } @@ -229,18 +229,18 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return Futures.immediateFuture(new TsKvLatestRemovingResult(query.getKey(), false)); } boolean isRemoved = false; + Long version = null; long ts = latest.getTs(); if (ts >= query.getStartTs() && ts < query.getEndTs()) { - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); - latestEntity.setEntityId(entityId.getId()); - latestEntity.setKey(keyDictionaryDao.getOrSaveKeyId(query.getKey())); - tsKvLatestRepository.delete(latestEntity); + version = transactionTemplate.execute(status -> jdbcTemplate.query("DELETE FROM ts_kv_latest WHERE entity_id = ? " + + "AND key = ? RETURNING nextval('ts_kv_latest_version_seq')", + rs -> rs.next() ? rs.getLong(1) : null, entityId.getId(), keyDictionaryDao.getOrSaveKeyId(query.getKey()))); isRemoved = true; if (query.getRewriteLatestIfDeleted()) { return getNewLatestEntryFuture(tenantId, entityId, query); } } - return Futures.immediateFuture(new TsKvLatestRemovingResult(query.getKey(), isRemoved)); + return Futures.immediateFuture(new TsKvLatestRemovingResult(query.getKey(), isRemoved, version)); }, MoreExecutors.directExecutor()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java index 50652d4f60..5e9e3369a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java @@ -31,7 +31,7 @@ public class SearchTsKvLatestRepository { public static final String FIND_ALL_BY_ENTITY_ID = "findAllByEntityId"; public static final String FIND_ALL_BY_ENTITY_ID_QUERY = "SELECT ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key, key_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue," + - " ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue, ts_kv_latest.ts AS ts FROM ts_kv_latest " + + " ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue, ts_kv_latest.ts AS ts, ts_kv_latest.version AS version FROM ts_kv_latest " + "INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)"; @PersistenceContext diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index f774f18dc7..01c91d4801 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -161,7 +161,7 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes var entryList = result.getData(); if (entryList.size() == 1) { TsKvEntry entry = entryList.get(0); - return Futures.transform(saveLatest(tenantId, entityId, entryList.get(0)), v -> new TsKvLatestRemovingResult(entry), MoreExecutors.directExecutor()); + return Futures.transform(saveLatest(tenantId, entityId, entryList.get(0)), v -> new TsKvLatestRemovingResult(entry, v), MoreExecutors.directExecutor()); } else { log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java index e9f9ab522c..851d66ae0c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java @@ -15,134 +15,41 @@ */ package org.thingsboard.server.dao.timeseries; -import jakarta.annotation.PostConstruct; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.NotImplementedException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.dao.InvalidDataAccessApiUsageException; -import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.connection.ReturnType; -import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; -import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; -import org.thingsboard.server.cache.TbCacheTransaction; -import org.thingsboard.server.cache.TbCacheValueWrapper; -import org.thingsboard.server.cache.TbJavaRedisSerializer; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.VersionedRedisTbCache; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.TsKvEntry; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("TsLatestCache") @Slf4j -public class TsLatestRedisCache extends RedisTbTransactionalCache { - - static final byte[] UPSERT_TS_LATEST_LUA_SCRIPT = StringRedisSerializer.UTF_8.serialize("" + - "redis.call('ZREMRANGEBYSCORE', KEYS[1], ARGV[1], ARGV[1]); " + - "redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2]); " + - "local current_size = redis.call('ZCARD', KEYS[1]); " + - "if current_size > 1 then" + - " redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -2) " + - "end;"); - static final byte[] UPSERT_TS_LATEST_SHA = StringRedisSerializer.UTF_8.serialize("24e226c3ea34e3e850113e8eb1f3cd2b88171988"); +public class TsLatestRedisCache extends VersionedRedisTbCache { public TsLatestRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJavaRedisSerializer<>()); - } - - @PostConstruct - public void init() { - try (var connection = getConnection(UPSERT_TS_LATEST_SHA)) { - log.debug("Loading LUA with expected SHA[{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), connection.getNativeConnection()); - String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT); - if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { - log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), sha, connection.getNativeConnection()); + super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>() { + @Override + public byte[] serialize(TsKvEntry tsKvEntry) throws SerializationException { + return KvProtoUtil.toTsKvProto(tsKvEntry.getTs(), tsKvEntry).toByteArray(); } - } catch (Throwable t) { - log.error("Error on Redis TS Latest cache init", t); - } - } - - @Override - public TbCacheValueWrapper get(TsLatestCacheKey key) { - log.debug("get [{}]", key); - return super.get(key); - } - - @Override - protected byte[] doGet(RedisConnection connection, byte[] rawKey) { - log.trace("doGet [{}][{}]", connection, rawKey); - Set values = connection.commands().zRange(rawKey, -1, -1); - return values == null ? null : values.stream().findFirst().orElse(null); - } - @Override - public void put(TsLatestCacheKey key, TsKvEntry value) { - log.trace("put [{}][{}]", key, value); - final byte[] rawKey = getRawKey(key); - try (var connection = getConnection(rawKey)) { - byte[] rawValue = getRawValue(value); - byte[] ts = StringRedisSerializer.UTF_8.serialize(String.valueOf(value.toTsValue().getTs())); - try { - connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue); - } catch (InvalidDataAccessApiUsageException e) { - log.debug("loading LUA [{}]", connection.getNativeConnection()); - String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT); - if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { - log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}]", new String(UPSERT_TS_LATEST_SHA), sha); - } + @Override + public TsKvEntry deserialize(TsLatestCacheKey key, byte[] bytes) throws SerializationException { try { - connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue); - } catch (InvalidDataAccessApiUsageException ignored) { - log.debug("Slowly executing eval instead of fast evalsha"); - connection.scriptingCommands().eval(UPSERT_TS_LATEST_LUA_SCRIPT, ReturnType.VALUE, 1, rawKey, ts, rawValue); + return KvProtoUtil.fromTsKvProto(TransportProtos.TsKvProto.parseFrom(bytes)); + } catch (InvalidProtocolBufferException e) { + throw new SerializationException(e.getMessage()); } - } - } - } - - @Override - public void evict(TsLatestCacheKey key) { - log.trace("evict [{}]", key); - final byte[] rawKey = getRawKey(key); - try (var connection = getConnection(rawKey)) { - connection.keyCommands().del(rawKey); - } - } - - @Override - public void putIfAbsent(TsLatestCacheKey key, TsKvEntry value) { - log.trace("putIfAbsent [{}][{}]", key, value); - throw new NotImplementedException("putIfAbsent is not supported by TsLatestRedisCache"); - } - - @Override - public void evict(Collection keys) { - throw new NotImplementedException("evict by many keys is not supported by TsLatestRedisCache"); - } - - @Override - public void evictOrPut(TsLatestCacheKey key, TsKvEntry value) { - throw new NotImplementedException("evictOrPut is not supported by TsLatestRedisCache"); - } - - @Override - public TbCacheTransaction newTransactionForKey(TsLatestCacheKey key) { - throw new NotImplementedException("newTransactionForKey is not supported by TsLatestRedisCache"); + }); } - - @Override - public TbCacheTransaction newTransactionForKeys(List keys) { - throw new NotImplementedException("newTransactionForKeys is not supported by TsLatestRedisCache"); - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index ad2470a9b1..0603f4e3f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -59,9 +59,6 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { private static final String OLD_VALUE = "OLD VALUE"; private static final String NEW_VALUE = "NEW VALUE"; - @Autowired - private VersionedTbCache cache; - @Autowired private AttributesService attributesService; From 84ea3b4afcbc30adb78b9be5453183059050f7cd Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 3 Jul 2024 17:51:52 +0200 Subject: [PATCH 26/42] test fixes --- .../dao/service/timeseries/BaseTimeseriesServiceTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index d32933684a..e4cc64d47c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -23,6 +23,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.DeviceId; @@ -74,6 +75,9 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Autowired EntityViewService entityViewService; + @Value("${database.ts.type}") + String databaseTsLatestType; + protected static final int MAX_TIMEOUT = 30; private static final String STRING_KEY = "stringKey"; @@ -161,6 +165,10 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { @Test public void testFindLatestOpt_givenSaveWithHistoricalNonOrderedTS() throws Exception { + if (databaseTsLatestType.equals("cassandra")) { + return; + } + save(tenantId, deviceId, toTsEntry(TS - 1, stringKvEntry)); save(tenantId, deviceId, toTsEntry(TS, stringKvEntry)); save(tenantId, deviceId, toTsEntry(TS - 10, stringKvEntry)); From 77a420b6d6aab641226c6cbea5b1a8bda17d8976 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 4 Jul 2024 19:02:19 +0200 Subject: [PATCH 27/42] fixed versioned caffeine cache --- .../server/cache/VersionedCaffeineTbCache.java | 11 +++++++---- .../thingsboard/server/common/data/util/TbPair.java | 7 ------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java index e084668b90..f2ac3c36d2 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java @@ -35,7 +35,11 @@ public abstract class VersionedCaffeineTbCache get(K key) { - return SimpleTbCacheValueWrapper.wrap(doGet(key).getSecond()); + TbPair versionValuePair = doGet(key); + if (versionValuePair != null) { + return SimpleTbCacheValueWrapper.wrap(versionValuePair.getSecond()); + } + return null; } @Override @@ -52,8 +56,7 @@ public abstract class VersionedCaffeineTbCache versionValuePair = doGet(key); - Long currentVersion = versionValuePair.getFirst(); - if (currentVersion == null || version > currentVersion) { + if (versionValuePair == null || version > versionValuePair.getFirst()) { cacheManager.getCache(cacheName).put(key, TbPair.of(version, value)); } } finally { @@ -63,7 +66,7 @@ public abstract class VersionedCaffeineTbCache doGet(K key) { Cache.ValueWrapper source = cacheManager.getCache(cacheName).get(key); - return source == null ? TbPair.emptyPair() : (TbPair) source.get(); + return source == null ? null : (TbPair) source.get(); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java index 315131bdad..6e8b2e5696 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbPair.java @@ -21,17 +21,10 @@ import lombok.Data; @Data @AllArgsConstructor public class TbPair { - public static final TbPair EMPTY = new TbPair<>(null, null); - private S first; private T second; public static TbPair of(S first, T second) { return new TbPair<>(first, second); } - - @SuppressWarnings("unchecked") - public static TbPair emptyPair() { - return (TbPair) EMPTY; - } } From cf821515fb215a9a6fb911f8a896019dacdc7413 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 9 Jul 2024 13:17:25 +0300 Subject: [PATCH 28/42] Refactoring for versioned caches --- .../cache/CaffeineTbCacheTransaction.java | 2 +- .../cache/CaffeineTbTransactionalCache.java | 25 +++++---- .../server/cache/RedisTbCacheTransaction.java | 5 +- .../cache/RedisTbTransactionalCache.java | 6 ++- .../server/cache/TbCacheTransaction.java | 2 +- .../server/cache/TbTransactionalCache.java | 4 +- .../cache/VersionedCaffeineTbCache.java | 29 ++++------ .../server/cache/VersionedRedisTbCache.java | 53 +++++++++---------- .../server/cache/VersionedTbCache.java | 3 +- .../resources/application-test.properties | 13 ++++- 10 files changed, 77 insertions(+), 65 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java index 54465b0b50..03d9099541 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java @@ -41,7 +41,7 @@ public class CaffeineTbCacheTransaction pendingPuts = new LinkedHashMap<>(); @Override - public void putIfAbsent(K key, V value) { + public void put(K key, V value) { pendingPuts.put(key, value); } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java index 9c01b47b88..e0a1457d72 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java @@ -17,6 +17,7 @@ package org.thingsboard.server.cache; import lombok.Getter; import lombok.RequiredArgsConstructor; +import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import java.io.Serializable; @@ -26,6 +27,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.locks.Lock; @@ -34,17 +36,22 @@ import java.util.concurrent.locks.ReentrantLock; @RequiredArgsConstructor public abstract class CaffeineTbTransactionalCache implements TbTransactionalCache { - private final CacheManager cacheManager; @Getter - private final String cacheName; - - private final Lock lock = new ReentrantLock(); + protected final String cacheName; + protected final Cache cache; + protected final Lock lock = new ReentrantLock(); private final Map> objectTransactions = new HashMap<>(); private final Map> transactions = new HashMap<>(); + public CaffeineTbTransactionalCache(CacheManager cacheManager, String cacheName) { + this.cacheName = cacheName; + this.cache = Optional.ofNullable(cacheManager.getCache(cacheName)) + .orElseThrow(() -> new IllegalArgumentException("Cache '" + cacheName + "' is not configured")); + } + @Override public TbCacheValueWrapper get(K key) { - return SimpleTbCacheValueWrapper.wrap(cacheManager.getCache(cacheName).get(key)); + return SimpleTbCacheValueWrapper.wrap(cache.get(key)); } @Override @@ -52,7 +59,7 @@ public abstract class CaffeineTbTransactionalCache newTransaction(List keys) { @@ -181,7 +188,7 @@ public abstract class CaffeineTbTransactionalCache transactionsIds = objectTransactions.get(key); if (transactionsIds != null) { for (UUID otherTrId : transactionsIds) { diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java index 0cb2d661db..fb852493ce 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java @@ -18,7 +18,6 @@ package org.thingsboard.server.cache; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.redis.connection.RedisConnection; -import org.springframework.data.redis.connection.RedisStringCommands; import java.io.Serializable; import java.util.Objects; @@ -31,8 +30,8 @@ public class RedisTbCacheTransaction { - void putIfAbsent(K key, V value); + void put(K key, V value); boolean commit(); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java index be0b38b65e..7507ef99ff 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java @@ -68,7 +68,7 @@ public interface TbTransactionalCache implements VersionedTbCache { +public abstract class VersionedCaffeineTbCache extends CaffeineTbTransactionalCache implements VersionedTbCache { - private final CacheManager cacheManager; - private final String cacheName; - - private final Lock lock = new ReentrantLock(); + public VersionedCaffeineTbCache(CacheManager cacheManager, String cacheName) { + super(cacheManager, cacheName); + } @Override public TbCacheValueWrapper get(K key) { @@ -57,7 +52,8 @@ public abstract class VersionedCaffeineTbCache versionValuePair = doGet(key); if (versionValuePair == null || version > versionValuePair.getFirst()) { - cacheManager.getCache(cacheName).put(key, TbPair.of(version, value)); + failAllTransactionsByKey(key); + cache.put(key, TbPair.of(version, value)); } } finally { lock.unlock(); @@ -65,7 +61,7 @@ public abstract class VersionedCaffeineTbCache doGet(K key) { - Cache.ValueWrapper source = cacheManager.getCache(cacheName).get(key); + Cache.ValueWrapper source = cache.get(key); return source == null ? null : (TbPair) source.get(); } @@ -73,7 +69,8 @@ public abstract class VersionedCaffeineTbCache extends RedisTbTransactionalCache implements VersionedTbCache { @@ -47,7 +46,7 @@ public abstract class VersionedRedisTbCacheI8", newVersion) .. newValue redis.call('SET', key, newValueWithVersion, 'EX', expiration) end - + local function bytes_to_number(bytes) local n = 0 for i = 1, 8 do @@ -102,33 +101,44 @@ public abstract class VersionedRedisTbCache newTransactionForKey(K key) { - throw new NotImplementedException("newTransactionForKey is not supported by versioned cache"); - } - - @Override - public TbCacheTransaction newTransactionForKeys(List keys) { - throw new NotImplementedException("newTransactionForKeys is not supported by versioned cache"); - } - } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java index aaec7a47be..8fcfcff8d7 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.HasVersion; import java.io.Serializable; -public interface VersionedTbCache { +public interface VersionedTbCache extends TbTransactionalCache { TbCacheValueWrapper get(K key); @@ -30,4 +30,5 @@ public interface VersionedTbCache Date: Tue, 9 Jul 2024 14:13:14 +0300 Subject: [PATCH 29/42] Fix Caffeine's getAndPutInTransaction; add compute method to VersionedTbCache --- .../server/cache/CaffeineTbCacheTransaction.java | 2 +- .../cache/CaffeineTbTransactionalCache.java | 4 ++-- .../server/cache/VersionedCaffeineTbCache.java | 11 ++++++++++- .../server/cache/VersionedRedisTbCache.java | 6 ------ .../server/cache/VersionedTbCache.java | 15 +++++++++++++++ 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java index 03d9099541..47778c96b6 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java @@ -38,7 +38,7 @@ public class CaffeineTbCacheTransaction pendingPuts = new LinkedHashMap<>(); + private final Map pendingPuts = new LinkedHashMap<>(); @Override public void put(K key, V value) { diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java index e0a1457d72..d2ea960e68 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java @@ -116,7 +116,7 @@ public abstract class CaffeineTbTransactionalCache pendingPuts) { + public boolean commit(UUID trId, Map pendingPuts) { lock.lock(); try { var tr = transactions.get(trId); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java index a1f85611a7..86849586cc 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java @@ -53,7 +53,7 @@ public abstract class VersionedCaffeineTbCache versionValuePair = doGet(key); if (versionValuePair == null || version > versionValuePair.getFirst()) { failAllTransactionsByKey(key); - cache.put(key, TbPair.of(version, value)); + cache.put(key, wrapValue(value, version)); } } finally { lock.unlock(); @@ -84,4 +84,13 @@ public abstract class VersionedCaffeineTbCache wrapValue(V value, Long version) { + return TbPair.of(version, value); + } + } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java index b7722f90fd..bfb19ad01a 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.HasVersion; import java.io.Serializable; import java.util.Arrays; -import java.util.Collection; @Slf4j public abstract class VersionedRedisTbCache extends RedisTbTransactionalCache implements VersionedTbCache { @@ -156,11 +155,6 @@ public abstract class VersionedRedisTbCache keys) { - throw new NotImplementedException("evict by many keys is not supported by versioned cache"); - } - @Override public void evictOrPut(K key, V value) { throw new NotImplementedException("evictOrPut is not supported by versioned cache"); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java index 8fcfcff8d7..4d600bd363 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedTbCache.java @@ -18,17 +18,32 @@ package org.thingsboard.server.cache; import org.thingsboard.server.common.data.HasVersion; import java.io.Serializable; +import java.util.Collection; +import java.util.Optional; +import java.util.function.Supplier; public interface VersionedTbCache extends TbTransactionalCache { TbCacheValueWrapper get(K key); + default V get(K key, Supplier supplier) { + return Optional.ofNullable(get(key)) + .map(TbCacheValueWrapper::get) + .orElseGet(() -> { + V value = supplier.get(); + put(key, value); + return value; + }); + } + void put(K key, V value); void put(K key, V value, Long version); void evict(K key); + void evict(Collection keys); + void evict(K key, Long version); } From d6367c9680188b213826e3dcf252776ae90f0131 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 9 Jul 2024 15:19:01 +0200 Subject: [PATCH 30/42] added version to timeseries and attribute protos --- .../cache/VersionedCaffeineTbCache.java | 7 +- .../server/cache/VersionedRedisTbCache.java | 18 +-- .../server/cache/VersionedTbCache.java | 2 - .../common/data/kv/BaseAttributeKvEntry.java | 39 +----- .../server/common/data/kv/BasicTsKvEntry.java | 34 +----- .../server/common/util/KvProtoUtil.java | 9 +- .../server/common/util/ProtoUtils.java | 94 ++++++++------- common/proto/src/main/proto/queue.proto | 2 + .../dao/attributes/AttributeRedisCache.java | 54 +-------- .../attributes/CachedAttributesService.java | 22 ++-- .../CachedRedisSqlTimeseriesLatestDao.java | 12 +- .../dao/timeseries/TsLatestRedisCache.java | 2 +- .../attributes/BaseAttributesServiceTest.java | 23 ++-- .../sql/AttributeCacheServiceSqlTest.java | 114 ++++++++++++++++++ .../timeseries/BaseTimeseriesServiceTest.java | 22 +++- 15 files changed, 253 insertions(+), 201 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/attributes/sql/AttributeCacheServiceSqlTest.java diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java index f2ac3c36d2..8ed4928d97 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedCaffeineTbCache.java @@ -45,11 +45,10 @@ public abstract class VersionedCaffeineTbCacheI8", newVersion) .. newValue redis.call('SET', key, newValueWithVersion, 'EX', expiration) end - + local function bytes_to_number(bytes) local n = 0 for i = 1, 8 do @@ -96,13 +96,15 @@ public abstract class VersionedRedisTbCache>> 32)); - result = 31 * result + kv.hashCode(); - return result; - } - - @Override - public String toString() { - return "BaseAttributeKvEntry{" + - "lastUpdateTs=" + lastUpdateTs + - ", kv=" + kv + - '}'; - } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java index ab39498ae0..82f96ea966 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicTsKvEntry.java @@ -16,9 +16,12 @@ package org.thingsboard.server.common.data.kv; import jakarta.validation.Valid; +import lombok.Data; + import java.util.Objects; import java.util.Optional; +@Data public class BasicTsKvEntry implements TsKvEntry { private static final int MAX_CHARS_PER_DATA_POINT = 512; protected final long ts; @@ -79,33 +82,6 @@ public class BasicTsKvEntry implements TsKvEntry { return kv.getValue(); } - @Override - public long getTs() { - return ts; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof BasicTsKvEntry)) return false; - BasicTsKvEntry that = (BasicTsKvEntry) o; - return getTs() == that.getTs() && - Objects.equals(kv, that.kv); - } - - @Override - public int hashCode() { - return Objects.hash(getTs(), kv); - } - - @Override - public String toString() { - return "BasicTsKvEntry{" + - "ts=" + ts + - ", kv=" + kv + - '}'; - } - @Override public String getValueAsString() { return kv.getValueAsString(); @@ -127,8 +103,4 @@ public class BasicTsKvEntry implements TsKvEntry { return Math.max(1, (length + MAX_CHARS_PER_DATA_POINT - 1) / MAX_CHARS_PER_DATA_POINT); } - @Override - public Long getVersion() { - return version; - } } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java b/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java index 74674e1e45..4b206fd804 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java @@ -86,8 +86,15 @@ public class KvProtoUtil { .setKv(KvProtoUtil.toKeyValueTypeProto(kvEntry)).build(); } + public static TransportProtos.TsKvProto toTsKvProto(long ts, KvEntry kvEntry, long version) { + return TransportProtos.TsKvProto.newBuilder() + .setTs(ts) + .setVersion(version) + .setKv(KvProtoUtil.toKeyValueTypeProto(kvEntry)).build(); + } + public static TsKvEntry fromTsKvProto(TransportProtos.TsKvProto proto) { - return new BasicTsKvEntry(proto.getTs(), fromTsKvProto(proto.getKv())); + return new BasicTsKvEntry(proto.getTs(), fromTsKvProto(proto.getKv()), proto.getVersion()); } public static TransportProtos.KeyValueProto toKeyValueTypeProto(KvEntry kvEntry) { diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index e3d587e915..db1c1fe2e5 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -17,7 +17,9 @@ package org.thingsboard.server.common.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; @@ -256,42 +258,47 @@ public class ProtoUtils { if (msg.getValues() != null) { for (AttributeKvEntry attributeKvEntry : msg.getValues()) { - TransportProtos.AttributeValueProto.Builder attributeValueBuilder = TransportProtos.AttributeValueProto.newBuilder() - .setLastUpdateTs(attributeKvEntry.getLastUpdateTs()) - .setKey(attributeKvEntry.getKey()); - switch (attributeKvEntry.getDataType()) { - case BOOLEAN -> { - attributeKvEntry.getBooleanValue().ifPresent(attributeValueBuilder::setBoolV); - attributeValueBuilder.setHasV(attributeKvEntry.getBooleanValue().isPresent()); - attributeValueBuilder.setType(TransportProtos.KeyValueType.BOOLEAN_V); - } - case STRING -> { - attributeKvEntry.getStrValue().ifPresent(attributeValueBuilder::setStringV); - attributeValueBuilder.setHasV(attributeKvEntry.getStrValue().isPresent()); - attributeValueBuilder.setType(TransportProtos.KeyValueType.STRING_V); - } - case DOUBLE -> { - attributeKvEntry.getDoubleValue().ifPresent(attributeValueBuilder::setDoubleV); - attributeValueBuilder.setHasV(attributeKvEntry.getDoubleValue().isPresent()); - attributeValueBuilder.setType(TransportProtos.KeyValueType.DOUBLE_V); - } - case LONG -> { - attributeKvEntry.getLongValue().ifPresent(attributeValueBuilder::setLongV); - attributeValueBuilder.setHasV(attributeKvEntry.getLongValue().isPresent()); - attributeValueBuilder.setType(TransportProtos.KeyValueType.LONG_V); - } - case JSON -> { - attributeKvEntry.getJsonValue().ifPresent(attributeValueBuilder::setJsonV); - attributeValueBuilder.setHasV(attributeKvEntry.getJsonValue().isPresent()); - attributeValueBuilder.setType(TransportProtos.KeyValueType.JSON_V); - } - } - builder.addValues(attributeValueBuilder.build()); + builder.addValues(toProto(attributeKvEntry)); } } return builder.build(); } + public static TransportProtos.AttributeValueProto toProto(AttributeKvEntry attributeKvEntry) { + TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder() + .setLastUpdateTs(attributeKvEntry.getLastUpdateTs()) + .setKey(attributeKvEntry.getKey()); + switch (attributeKvEntry.getDataType()) { + case BOOLEAN: + attributeKvEntry.getBooleanValue().ifPresent(builder::setBoolV); + builder.setHasV(attributeKvEntry.getBooleanValue().isPresent()); + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V); + break; + case STRING: + attributeKvEntry.getStrValue().ifPresent(builder::setStringV); + builder.setHasV(attributeKvEntry.getStrValue().isPresent()); + builder.setType(TransportProtos.KeyValueType.STRING_V); + break; + case DOUBLE: + attributeKvEntry.getDoubleValue().ifPresent(builder::setDoubleV); + builder.setHasV(attributeKvEntry.getDoubleValue().isPresent()); + builder.setType(TransportProtos.KeyValueType.DOUBLE_V); + break; + case LONG: + attributeKvEntry.getLongValue().ifPresent(builder::setLongV); + builder.setHasV(attributeKvEntry.getLongValue().isPresent()); + builder.setType(TransportProtos.KeyValueType.LONG_V); + break; + case JSON: + attributeKvEntry.getJsonValue().ifPresent(builder::setJsonV); + builder.setHasV(attributeKvEntry.getJsonValue().isPresent()); + builder.setType(TransportProtos.KeyValueType.JSON_V); + break; + } + builder.setVersion(attributeKvEntry.getVersion()); + return builder.build(); + } + private static ToDeviceActorNotificationMsg fromProto(TransportProtos.DeviceAttributesEventMsgProto proto) { return new DeviceAttributesEventNotificationMsg( TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), @@ -500,20 +507,25 @@ public class ProtoUtils { } List result = new ArrayList<>(); for (TransportProtos.AttributeValueProto kvEntry : valuesList) { - boolean hasValue = kvEntry.getHasV(); - KvEntry entry = switch (kvEntry.getType()) { - case BOOLEAN_V -> new BooleanDataEntry(kvEntry.getKey(), hasValue ? kvEntry.getBoolV() : null); - case LONG_V -> new LongDataEntry(kvEntry.getKey(), hasValue ? kvEntry.getLongV() : null); - case DOUBLE_V -> new DoubleDataEntry(kvEntry.getKey(), hasValue ? kvEntry.getDoubleV() : null); - case STRING_V -> new StringDataEntry(kvEntry.getKey(), hasValue ? kvEntry.getStringV() : null); - case JSON_V -> new JsonDataEntry(kvEntry.getKey(), hasValue ? kvEntry.getJsonV() : null); - default -> null; - }; - result.add(new BaseAttributeKvEntry(kvEntry.getLastUpdateTs(), entry)); + result.add(fromProto(kvEntry)); } return result; } + public static AttributeKvEntry fromProto(TransportProtos.AttributeValueProto proto) { + boolean hasValue = proto.getHasV(); + String key = proto.getKey(); + KvEntry entry = switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, hasValue ? proto.getBoolV() : null); + case LONG_V -> new LongDataEntry(key, hasValue ? proto.getLongV() : null); + case DOUBLE_V -> new DoubleDataEntry(key, hasValue ? proto.getDoubleV() : null); + case STRING_V -> new StringDataEntry(key, hasValue ? proto.getStringV() : null); + case JSON_V -> new JsonDataEntry(key, hasValue ? proto.getJsonV() : null); + default -> null; + }; + return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.getVersion()); + } + public static TransportProtos.DeviceProto toProto(Device device) { var builder = TransportProtos.DeviceProto.newBuilder() .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 581b993120..9b93812708 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -157,11 +157,13 @@ message AttributeValueProto { string string_v = 7; string json_v = 8; optional string key = 9; + int64 version = 10; } message TsKvProto { int64 ts = 1; KeyValueProto kv = 2; + int64 version = 3; } message TsKvListProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java index a6f21f9c3a..2b182c2b92 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java @@ -26,15 +26,8 @@ import org.thingsboard.server.cache.TbRedisSerializer; import org.thingsboard.server.cache.VersionedRedisTbCache; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("AttributeCache") @@ -44,54 +37,13 @@ public class AttributeRedisCache extends VersionedRedisTbCache() { @Override public byte[] serialize(AttributeKvEntry attributeKvEntry) throws SerializationException { - AttributeValueProto.Builder builder = AttributeValueProto.newBuilder() - .setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); - switch (attributeKvEntry.getDataType()) { - case BOOLEAN: - attributeKvEntry.getBooleanValue().ifPresent(builder::setBoolV); - builder.setHasV(attributeKvEntry.getBooleanValue().isPresent()); - builder.setType(KeyValueType.BOOLEAN_V); - break; - case STRING: - attributeKvEntry.getStrValue().ifPresent(builder::setStringV); - builder.setHasV(attributeKvEntry.getStrValue().isPresent()); - builder.setType(KeyValueType.STRING_V); - break; - case DOUBLE: - attributeKvEntry.getDoubleValue().ifPresent(builder::setDoubleV); - builder.setHasV(attributeKvEntry.getDoubleValue().isPresent()); - builder.setType(KeyValueType.DOUBLE_V); - break; - case LONG: - attributeKvEntry.getLongValue().ifPresent(builder::setLongV); - builder.setHasV(attributeKvEntry.getLongValue().isPresent()); - builder.setType(KeyValueType.LONG_V); - break; - case JSON: - attributeKvEntry.getJsonValue().ifPresent(builder::setJsonV); - builder.setHasV(attributeKvEntry.getJsonValue().isPresent()); - builder.setType(KeyValueType.JSON_V); - break; - - } - return builder.build().toByteArray(); + return ProtoUtils.toProto(attributeKvEntry).toByteArray(); } @Override public AttributeKvEntry deserialize(AttributeCacheKey key, byte[] bytes) throws SerializationException { try { - AttributeValueProto proto = AttributeValueProto.parseFrom(bytes); - boolean hasValue = proto.getHasV(); - KvEntry entry = switch (proto.getType()) { - case BOOLEAN_V -> new BooleanDataEntry(key.getKey(), hasValue ? proto.getBoolV() : null); - case LONG_V -> new LongDataEntry(key.getKey(), hasValue ? proto.getLongV() : null); - case DOUBLE_V -> new DoubleDataEntry(key.getKey(), hasValue ? proto.getDoubleV() : null); - case STRING_V -> new StringDataEntry(key.getKey(), hasValue ? proto.getStringV() : null); - case JSON_V -> new JsonDataEntry(key.getKey(), hasValue ? proto.getJsonV() : null); - default -> - throw new InvalidProtocolBufferException("Unrecognized type: " + proto.getType() + " !"); - }; - return new BaseAttributeKvEntry(proto.getLastUpdateTs(), entry); + return ProtoUtils.fromProto(AttributeValueProto.parseFrom(bytes)); } catch (InvalidProtocolBufferException e) { throw new SerializationException(e.getMessage()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index d31ff9db79..3de90355ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; @@ -159,7 +160,7 @@ public class CachedAttributesService implements AttributesService { log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); for (AttributeKvEntry foundInDbAttribute : result) { - put(entityId, scope, foundInDbAttribute, foundInDbAttribute.getVersion()); + put(entityId, scope, foundInDbAttribute); notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); } for (String key : notFoundAttributeKeys) { @@ -218,8 +219,7 @@ public class CachedAttributesService implements AttributesService { public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> put(entityId, scope, attribute, version), cacheExecutor); + return doSave(tenantId, entityId, scope, attribute); } @Override @@ -234,19 +234,25 @@ public class CachedAttributesService implements AttributesService { List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { - ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - futures.add(Futures.transform(future, version -> put(entityId, scope, attribute, version), cacheExecutor)); + futures.add(doSave(tenantId, entityId, scope, attribute)); } return Futures.allAsList(futures); } - private Long put(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute, Long version) { + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + return Futures.transform(future, version -> { + put(entityId, scope, new BaseAttributeKvEntry(((BaseAttributeKvEntry)attribute).getKv(), attribute.getLastUpdateTs(), version)); + return version; + }, cacheExecutor); + } + + private void put(EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { String key = attribute.getKey(); log.trace("[{}][{}][{}] Before cache put: {}", entityId, scope, key, attribute); - cache.put(new AttributeCacheKey(scope, entityId, key), attribute, version); + cache.put(new AttributeCacheKey(scope, entityId, key), attribute); log.trace("[{}][{}][{}] after cache put.", entityId, scope, key); - return version; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index 42dc149342..0be078a89f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; @@ -66,9 +67,9 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { ListenableFuture future = sqlDao.saveLatest(tenantId, entityId, tsKvEntry); - future = Futures.transform(future, x -> { - cache.put(new TsLatestCacheKey(entityId, tsKvEntry.getKey()), tsKvEntry, x); - return x; + future = Futures.transform(future, version -> { + cache.put(new TsLatestCacheKey(entityId, tsKvEntry.getKey()), new BasicTsKvEntry(tsKvEntry.getTs(), ((BasicTsKvEntry) tsKvEntry).getKv(), version)); + return version; }, cacheExecutorService); if (log.isTraceEnabled()) { @@ -94,8 +95,9 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries if (x.isRemoved()) { TsLatestCacheKey key = new TsLatestCacheKey(entityId, query.getKey()); Long version = x.getVersion(); - if (x.getData() != null) { - cache.put(key, x.getData(), version); + TsKvEntry newTsKvEntry = x.getData(); + if (newTsKvEntry != null) { + cache.put(key, new BasicTsKvEntry(newTsKvEntry.getTs(), ((BasicTsKvEntry) newTsKvEntry).getKv(), version)); } else { cache.evict(key, version); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java index 851d66ae0c..241132b4c8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java @@ -39,7 +39,7 @@ public class TsLatestRedisCache extends VersionedRedisTbCache() { @Override public byte[] serialize(TsKvEntry tsKvEntry) throws SerializationException { - return KvProtoUtil.toTsKvProto(tsKvEntry.getTs(), tsKvEntry).toByteArray(); + return KvProtoUtil.toTsKvProto(tsKvEntry.getTs(), tsKvEntry, tsKvEntry.getVersion()).toByteArray(); } @Override diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index 0603f4e3f6..2d648f9db3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -26,7 +26,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,7 +33,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.dao.attributes.AttributeCacheKey; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.service.AbstractServiceTest; @@ -74,7 +72,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attr)).get(); Optional saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, attr.getKey()).get(); Assert.assertTrue(saved.isPresent()); - Assert.assertEquals(attr, saved.get()); + equalsIgnoreVersion(attr, saved.get()); } @Test @@ -87,14 +85,15 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { Optional saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, attrOld.getKey()).get(); Assert.assertTrue(saved.isPresent()); - Assert.assertEquals(attrOld, saved.get()); + equalsIgnoreVersion(attrOld, saved.get()); KvEntry attrNewValue = new StringDataEntry("attribute1", "value2"); AttributeKvEntry attrNew = new BaseAttributeKvEntry(attrNewValue, 73L); attributesService.save(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, Collections.singletonList(attrNew)).get(); saved = attributesService.find(SYSTEM_TENANT_ID, deviceId, AttributeScope.CLIENT_SCOPE, attrOld.getKey()).get(); - Assert.assertEquals(attrNew, saved.get()); + Assert.assertTrue(saved.isPresent()); + equalsIgnoreVersion(attrNew, saved.get()); } @Test @@ -117,8 +116,8 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { Assert.assertNotNull(saved); Assert.assertEquals(2, saved.size()); - Assert.assertEquals(attrANew, saved.get(0)); - Assert.assertEquals(attrBNew, saved.get(1)); + equalsIgnoreVersion(attrANew, saved.get(0)); + equalsIgnoreVersion(attrBNew, saved.get(1)); } @Test @@ -253,6 +252,11 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { })); futures.add(pool.submit(() -> saveAttribute(tenantId, deviceId, scope, key, NEW_VALUE))); Futures.allAsList(futures).get(10, TimeUnit.SECONDS); + + String attributeValue = getAttributeValue(tenantId, deviceId, scope, key); + if (!NEW_VALUE.equals(attributeValue)) { + System.out.println(); + } Assert.assertEquals(NEW_VALUE, getAttributeValue(tenantId, deviceId, scope, key)); } @@ -309,5 +313,10 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { } } + private void equalsIgnoreVersion(AttributeKvEntry expected, AttributeKvEntry actual) { + Assert.assertEquals(expected.getKey(), actual.getKey()); + Assert.assertEquals(expected.getValue(), actual.getValue()); + Assert.assertEquals(expected.getLastUpdateTs(), actual.getLastUpdateTs()); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/sql/AttributeCacheServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/sql/AttributeCacheServiceSqlTest.java new file mode 100644 index 0000000000..95e3e15adc --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/sql/AttributeCacheServiceSqlTest.java @@ -0,0 +1,114 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.attributes.sql; + +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.cache.TbCacheValueWrapper; +import org.thingsboard.server.cache.VersionedTbCache; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.attributes.AttributeCacheKey; +import org.thingsboard.server.dao.service.AbstractServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +@DaoSqlTest +public class AttributeCacheServiceSqlTest extends AbstractServiceTest { + + private static final String TEST_KEY = "key"; + private static final String TEST_VALUE = "value"; + private static final DeviceId DEVICE_ID = new DeviceId(UUID.randomUUID()); + + @Autowired + VersionedTbCache cache; + + @Test + public void testPutAndGet() { + AttributeCacheKey testKey = new AttributeCacheKey(AttributeScope.CLIENT_SCOPE, DEVICE_ID, TEST_KEY); + AttributeKvEntry testValue = new BaseAttributeKvEntry(new StringDataEntry(TEST_KEY, TEST_VALUE), 1, 1L); + cache.put(testKey, testValue); + + TbCacheValueWrapper wrapper = cache.get(testKey); + assertNotNull(wrapper); + + assertEquals(testValue, wrapper.get()); + + AttributeKvEntry testValue2 = new BaseAttributeKvEntry(new StringDataEntry(TEST_KEY, TEST_VALUE), 1, 2L); + cache.put(testKey, testValue2); + + wrapper = cache.get(testKey); + assertNotNull(wrapper); + + assertEquals(testValue2, wrapper.get()); + + AttributeKvEntry testValue3 = new BaseAttributeKvEntry(new StringDataEntry(TEST_KEY, TEST_VALUE), 1, 0L); + cache.put(testKey, testValue3); + + wrapper = cache.get(testKey); + assertNotNull(wrapper); + + assertEquals(testValue2, wrapper.get()); + + cache.evict(testKey); + } + + @Test + public void testEvictWithVersion() { + AttributeCacheKey testKey = new AttributeCacheKey(AttributeScope.CLIENT_SCOPE, DEVICE_ID, TEST_KEY); + AttributeKvEntry testValue = new BaseAttributeKvEntry(new StringDataEntry(TEST_KEY, TEST_VALUE), 1, 1L); + cache.put(testKey, testValue); + + TbCacheValueWrapper wrapper = cache.get(testKey); + assertNotNull(wrapper); + + assertEquals(testValue, wrapper.get()); + + cache.evict(testKey, 2L); + + wrapper = cache.get(testKey); + assertNotNull(wrapper); + + assertNull(wrapper.get()); + + cache.evict(testKey); + } + + @Test + public void testEvict() { + AttributeCacheKey testKey = new AttributeCacheKey(AttributeScope.CLIENT_SCOPE, DEVICE_ID, TEST_KEY); + AttributeKvEntry testValue = new BaseAttributeKvEntry(new StringDataEntry(TEST_KEY, TEST_VALUE), 1, 1L); + cache.put(testKey, testValue); + + TbCacheValueWrapper wrapper = cache.get(testKey); + assertNotNull(wrapper); + + assertEquals(testValue, wrapper.get()); + + cache.evict(testKey); + + wrapper = cache.get(testKey); + assertNull(wrapper); + } +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index e4cc64d47c..443052240a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -137,7 +137,12 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { toTsEntry(TS, booleanKvEntry)); Collections.sort(expected, Comparator.comparing(KvEntry::getKey)); - assertEquals(expected, tsList); + for (int i = 0; i < expected.size(); i++) { + var expectedEntry = expected.get(i); + var actualEntry = tsList.get(i); + equalsIgnoreVersion(expectedEntry, actualEntry); + + } } private EntityView saveAndCreateEntityView(DeviceId deviceId, List timeseries) { @@ -160,7 +165,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findLatest(tenantId, deviceId, Collections.singleton(STRING_KEY)).get(MAX_TIMEOUT, TimeUnit.SECONDS); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(0)); + equalsIgnoreVersion(toTsEntry(TS, stringKvEntry), entries.get(0)); } @Test @@ -176,7 +181,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); assertThat(entryOpt).isNotNull().isPresent(); - Assert.assertEquals(toTsEntry(TS, stringKvEntry), entryOpt.orElse(null)); + equalsIgnoreVersion(toTsEntry(TS, stringKvEntry), entryOpt.get()); } @Test @@ -186,7 +191,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); assertThat(entryOpt).isNotNull().isPresent(); - Assert.assertEquals(toTsEntry(TS, new StringDataEntry(STRING_KEY, "new")), entryOpt.orElse(null)); + equalsIgnoreVersion(toTsEntry(TS, new StringDataEntry(STRING_KEY, "new")), entryOpt.get()); } public void testFindLatestOpt_givenSaveWithSameTSOverwriteTypeAndValue() throws Exception { @@ -209,7 +214,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Optional entryOpt = tsService.findLatest(tenantId, deviceId, STRING_KEY).get(MAX_TIMEOUT, TimeUnit.SECONDS); assertThat(entryOpt).isNotNull().isPresent(); - Assert.assertEquals(toTsEntry(TS, stringKvEntry), entryOpt.get()); + equalsIgnoreVersion(toTsEntry(TS, stringKvEntry), entryOpt.get()); } @Test @@ -239,7 +244,7 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { List entries = tsService.findLatest(tenantId, deviceId, Collections.singleton(STRING_KEY)).get(MAX_TIMEOUT, TimeUnit.SECONDS); Assert.assertEquals(1, entries.size()); - Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(0)); + equalsIgnoreVersion(toTsEntry(TS - 1, stringKvEntry), entries.get(0)); } @Test @@ -794,5 +799,10 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { return new BasicTsKvEntry(ts, entry); } + private static void equalsIgnoreVersion(TsKvEntry expected, TsKvEntry actual) { + assertEquals(expected.getKey(), actual.getKey()); + assertEquals(expected.getValue(), actual.getValue()); + assertEquals(expected.getTs(), actual.getTs()); + } } From 2336cba89a61b5632742c80a5fc7d99774995952 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 9 Jul 2024 16:50:07 +0200 Subject: [PATCH 31/42] version in proto should be optional --- .../server/common/util/KvProtoUtil.java | 15 ++++++++++----- .../server/common/util/ProtoUtils.java | 8 ++++++-- common/proto/src/main/proto/queue.proto | 4 ++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java b/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java index 4b206fd804..1be79a28cc 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/KvProtoUtil.java @@ -86,15 +86,20 @@ public class KvProtoUtil { .setKv(KvProtoUtil.toKeyValueTypeProto(kvEntry)).build(); } - public static TransportProtos.TsKvProto toTsKvProto(long ts, KvEntry kvEntry, long version) { - return TransportProtos.TsKvProto.newBuilder() + public static TransportProtos.TsKvProto toTsKvProto(long ts, KvEntry kvEntry, Long version) { + var builder = TransportProtos.TsKvProto.newBuilder() .setTs(ts) - .setVersion(version) - .setKv(KvProtoUtil.toKeyValueTypeProto(kvEntry)).build(); + .setKv(KvProtoUtil.toKeyValueTypeProto(kvEntry)); + + if (version != null) { + builder.setVersion(version); + } + + return builder.build(); } public static TsKvEntry fromTsKvProto(TransportProtos.TsKvProto proto) { - return new BasicTsKvEntry(proto.getTs(), fromTsKvProto(proto.getKv()), proto.getVersion()); + return new BasicTsKvEntry(proto.getTs(), fromTsKvProto(proto.getKv()), proto.hasVersion() ? proto.getVersion() : null); } public static TransportProtos.KeyValueProto toKeyValueTypeProto(KvEntry kvEntry) { diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index db1c1fe2e5..b9377ac215 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -295,7 +295,11 @@ public class ProtoUtils { builder.setType(TransportProtos.KeyValueType.JSON_V); break; } - builder.setVersion(attributeKvEntry.getVersion()); + + if (attributeKvEntry.getVersion() != null) { + builder.setVersion(attributeKvEntry.getVersion()); + } + return builder.build(); } @@ -523,7 +527,7 @@ public class ProtoUtils { case JSON_V -> new JsonDataEntry(key, hasValue ? proto.getJsonV() : null); default -> null; }; - return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.getVersion()); + return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } public static TransportProtos.DeviceProto toProto(Device device) { diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 9b93812708..3317391f01 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -157,13 +157,13 @@ message AttributeValueProto { string string_v = 7; string json_v = 8; optional string key = 9; - int64 version = 10; + optional int64 version = 10; } message TsKvProto { int64 ts = 1; KeyValueProto kv = 2; - int64 version = 3; + optional int64 version = 3; } message TsKvListProto { From 164185c0f34c095c8e2c24b598dd4b8b6331faec Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 9 Jul 2024 17:17:23 +0200 Subject: [PATCH 32/42] added upgrade script for attributes and latest ts --- .../main/data/upgrade/3.7.0/schema_update.sql | 23 +++++++++++++++++++ .../install/ThingsboardInstallService.java | 4 ++++ .../install/SqlDatabaseUpgradeService.java | 3 +++ .../update/DefaultCacheCleanupService.java | 6 ++++- 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 application/src/main/data/upgrade/3.7.0/schema_update.sql diff --git a/application/src/main/data/upgrade/3.7.0/schema_update.sql b/application/src/main/data/upgrade/3.7.0/schema_update.sql new file mode 100644 index 0000000000..0e10667545 --- /dev/null +++ b/application/src/main/data/upgrade/3.7.0/schema_update.sql @@ -0,0 +1,23 @@ +-- +-- Copyright © 2016-2024 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- UPDATE PUBLIC CUSTOMERS START + +CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; + +ALTER TABLE attribute_kv ADD COLUMN version bigint default 0; +ALTER TABLE ts_kv_latest ADD COLUMN version bigint default 0; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 4bef0d420a..85eb64ffba 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -137,6 +137,10 @@ public class ThingsboardInstallService { entityDatabaseSchemaService.createCustomerTitleUniqueConstraintIfNotExists(); systemDataLoaderService.updateDefaultNotificationConfigs(false); systemDataLoaderService.updateSecuritySettings(); + break; + case "3.7.0": + log.info("Upgrading ThingsBoard from version 3.7.0 to 3.7.1 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.7.0"); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 86d922c305..000c0536cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -121,6 +121,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService case "3.6.4": updateSchema("3.6.4", 3006004, "3.7.0", 3007000, null); break; + case "3.7.0": + updateSchema("3.7.0", 3007000, "3.7.1", 3007001, null); + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index 7fd92bc5c6..0085df35d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -61,6 +61,10 @@ public class DefaultCacheCleanupService implements CacheCleanupService { log.info("Clearing cache to upgrade from version 3.6.4 to 3.7.0"); clearAll(); break; + case "3.7.0": + log.info("Clearing cache to upgrade from version 3.7.0 to 3.7.1"); + clearAll(); + break; default: //Do nothing, since cache cleanup is optional. } @@ -81,7 +85,7 @@ public class DefaultCacheCleanupService implements CacheCleanupService { if (redisTemplate.isPresent()) { log.info("Flushing all caches"); redisTemplate.get().execute((RedisCallback) connection -> { - connection.flushAll(); + connection.serverCommands().flushAll(); return null; }); return; From 9515bdea1d8c519cdda71ffe741bd2070ad00c23 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 9 Jul 2024 17:33:00 +0200 Subject: [PATCH 33/42] updated LUA SHA --- .../org/thingsboard/server/cache/VersionedRedisTbCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java index d0b3afec9c..3ddcc53f38 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java @@ -68,7 +68,7 @@ public abstract class VersionedRedisTbCache valueSerializer) { super(cacheName, cacheSpecsMap, connectionFactory, configuration, valueSerializer); From 7945669303cf45b9a336a5aa44e0edf9f862239d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 10 Jul 2024 12:22:08 +0200 Subject: [PATCH 34/42] fixed sparkplug tests --- ...ctMqttV5ClientSparkplugConnectionTest.java | 21 +++++++++++++------ ...actMqttV5ClientSparkplugTelemetryTest.java | 17 +++++++++++++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/connection/AbstractMqttV5ClientSparkplugConnectionTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/connection/AbstractMqttV5ClientSparkplugConnectionTest.java index 603b42f6b6..561594f0bf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/connection/AbstractMqttV5ClientSparkplugConnectionTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/connection/AbstractMqttV5ClientSparkplugConnectionTest.java @@ -63,7 +63,9 @@ public abstract class AbstractMqttV5ClientSparkplugConnectionTest extends Abstra return finalFuture.get().get().isPresent(); }); TsKvEntry actualTsKvEntry = finalFuture.get().get().get(); - Assert.assertEquals(expectedTsKvEntry, actualTsKvEntry); + Assert.assertEquals(expectedTsKvEntry.getKey(), actualTsKvEntry.getKey()); + Assert.assertEquals(expectedTsKvEntry.getValue(), actualTsKvEntry.getValue()); + Assert.assertEquals(expectedTsKvEntry.getTs(), actualTsKvEntry.getTs()); } protected void processClientWithCorrectNodeAccessTokenWithoutNDEATH_Test() throws Exception { @@ -95,20 +97,27 @@ public abstract class AbstractMqttV5ClientSparkplugConnectionTest extends Abstra List devices = connectClientWithCorrectAccessTokenWithNDEATHCreatedDevices(cntDevices, ts); TsKvEntry tsKvEntry = new BasicTsKvEntry(ts, new StringDataEntry(messageName(STATE), ONLINE.name())); - AtomicReference>> finalFuture = new AtomicReference<>(); await(alias + messageName(STATE) + ", device: " + savedGateway.getName()) .atMost(40, TimeUnit.SECONDS) .until(() -> { - finalFuture.set(tsService.findAllLatest(tenantId, savedGateway.getId())); - return finalFuture.get().get().contains(tsKvEntry); + var foundEntry = tsService.findAllLatest(tenantId, savedGateway.getId()).get().stream() + .filter(tsKv -> tsKv.getKey().equals(tsKvEntry.getKey())) + .filter(tsKv -> tsKv.getValue().equals(tsKvEntry.getValue())) + .filter(tsKv -> tsKv.getTs() == tsKvEntry.getTs()) + .findFirst(); + return foundEntry.isPresent(); }); for (Device device : devices) { await(alias + messageName(STATE) + ", device: " + device.getName()) .atMost(40, TimeUnit.SECONDS) .until(() -> { - finalFuture.set(tsService.findAllLatest(tenantId, device.getId())); - return finalFuture.get().get().contains(tsKvEntry); + var foundEntry = tsService.findAllLatest(tenantId, device.getId()).get().stream() + .filter(tsKv -> tsKv.getKey().equals(tsKvEntry.getKey())) + .filter(tsKv -> tsKv.getValue().equals(tsKvEntry.getValue())) + .filter(tsKv -> tsKv.getTs() == tsKvEntry.getTs()) + .findFirst(); + return foundEntry.isPresent(); }); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/timeseries/AbstractMqttV5ClientSparkplugTelemetryTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/timeseries/AbstractMqttV5ClientSparkplugTelemetryTest.java index af34e8bf0e..c637ce5bb1 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/timeseries/AbstractMqttV5ClientSparkplugTelemetryTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/timeseries/AbstractMqttV5ClientSparkplugTelemetryTest.java @@ -78,7 +78,7 @@ public abstract class AbstractMqttV5ClientSparkplugTelemetryTest extends Abstrac finalFuture.set(tsService.findAllLatest(tenantId, savedGateway.getId())); return finalFuture.get().get().size() == (listTsKvEntry.size() + 1); }); - Assert.assertTrue("Actual tsKvEntrys is not containsAll Expected tsKvEntrys", finalFuture.get().get().containsAll(listTsKvEntry)); + Assert.assertTrue("Actual tsKvEntries is not containsAll Expected tsKvEntries", containsIgnoreVersion(finalFuture.get().get(), listTsKvEntry)); } protected void processClientWithCorrectAccessTokenPushNodeMetricBuildArraysPrimitiveSimple() throws Exception { @@ -107,7 +107,20 @@ public abstract class AbstractMqttV5ClientSparkplugTelemetryTest extends Abstrac finalFuture.set(tsService.findAllLatest(tenantId, savedGateway.getId())); return finalFuture.get().get().size() == (listTsKvEntry.size() + 1); }); - Assert.assertTrue("Actual tsKvEntrys is not containsAll Expected tsKvEntrys", finalFuture.get().get().containsAll(listTsKvEntry)); + Assert.assertTrue("Actual tsKvEntries is not containsAll Expected tsKvEntries", containsIgnoreVersion(finalFuture.get().get(), listTsKvEntry)); } + private static boolean containsIgnoreVersion(List expected, List actual) { + for (TsKvEntry actualEntry : actual) { + var found = expected.stream() + .filter(tsKv -> tsKv.getKey().equals(actualEntry.getKey())) + .filter(tsKv -> tsKv.getValue().equals(actualEntry.getValue())) + .filter(tsKv -> tsKv.getTs() == actualEntry.getTs()) + .findFirst(); + if (found.isEmpty()) { + return false; + } + } + return true; + } } From 31d2d14f6069735efb3455608b1822216f162c58 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 16 Jul 2024 10:29:58 +0300 Subject: [PATCH 35/42] Cache transaction support for versioned entities --- .../main/data/upgrade/3.7.0/schema_update.sql | 4 +- .../cache/CaffeineTbTransactionalCache.java | 5 +++ .../server/cache/RedisTbCacheTransaction.java | 2 +- .../cache/RedisTbTransactionalCache.java | 13 ++++-- .../server/cache/TbTransactionalCache.java | 6 ++- .../server/cache/VersionedRedisTbCache.java | 40 +++++++++++-------- .../server/cache/VersionedTbCache.java | 8 +++- 7 files changed, 52 insertions(+), 26 deletions(-) diff --git a/application/src/main/data/upgrade/3.7.0/schema_update.sql b/application/src/main/data/upgrade/3.7.0/schema_update.sql index 0e10667545..f964387359 100644 --- a/application/src/main/data/upgrade/3.7.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.7.0/schema_update.sql @@ -14,10 +14,12 @@ -- limitations under the License. -- --- UPDATE PUBLIC CUSTOMERS START +-- KV VERSIONING UPDATE START CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1000; CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; ALTER TABLE attribute_kv ADD COLUMN version bigint default 0; ALTER TABLE ts_kv_latest ADD COLUMN version bigint default 0; + +-- KV VERSIONING UPDATE END diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java index d2ea960e68..4ce6571f1c 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java @@ -54,6 +54,11 @@ public abstract class CaffeineTbTransactionalCache get(K key, boolean transactionMode) { + return get(key); + } + @Override public void put(K key, V value) { lock.lock(); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java index fb852493ce..3dcb6e878f 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java @@ -31,7 +31,7 @@ public class RedisTbCacheTransaction get(K key) { + return get(key, false); + } + + @Override + public TbCacheValueWrapper get(K key, boolean transactionMode) { try (var connection = connectionFactory.getConnection()) { byte[] rawKey = getRawKey(key); - byte[] rawValue = doGet(connection, rawKey); + byte[] rawValue = doGet(connection, rawKey, transactionMode); if (rawValue == null || rawValue.length == 0) { return null; } else if (Arrays.equals(rawValue, BINARY_NULL_VALUE)) { @@ -96,18 +101,18 @@ public abstract class RedisTbTransactionalCache get(K key); + TbCacheValueWrapper get(K key, boolean transactionMode); + void put(K key, V value); void putIfAbsent(K key, V value); @@ -60,7 +62,7 @@ public interface TbTransactionalCache dbCall, boolean cacheNullValue) { - TbCacheValueWrapper cacheValueWrapper = get(key); + TbCacheValueWrapper cacheValueWrapper = get(key, true); if (cacheValueWrapper != null) { return cacheValueWrapper.get(); } @@ -95,7 +97,7 @@ public interface TbTransactionalCache R getAndPutInTransaction(K key, Supplier dbCall, Function cacheValueToResult, Function dbValueToCacheValue, boolean cacheNullValue) { - TbCacheValueWrapper cacheValueWrapper = get(key); + TbCacheValueWrapper cacheValueWrapper = get(key, true); if (cacheValueWrapper != null) { var cacheValue = cacheValueWrapper.get(); return cacheValue == null ? null : cacheValueToResult.apply(cacheValue); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java index 3ddcc53f38..67b6c86f96 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/VersionedRedisTbCache.java @@ -88,31 +88,30 @@ public abstract class VersionedRedisTbCache get(K key); default V get(K key, Supplier supplier) { + return get(key, supplier, true); + } + + default V get(K key, Supplier supplier, boolean putToCache) { return Optional.ofNullable(get(key)) .map(TbCacheValueWrapper::get) .orElseGet(() -> { V value = supplier.get(); - put(key, value); + if (putToCache) { + put(key, value); + } return value; }); } From 80ef8405c1e4aea1988b5e5e8017c5e3be3dbc1d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 19 Jul 2024 18:58:26 +0200 Subject: [PATCH 36/42] fixed clusterdown --- .../dao/AbstractRedisClusterContainer.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java index a95f4b8c17..90ff4fbc34 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisClusterContainer.java @@ -21,7 +21,6 @@ import org.junit.rules.ExternalResource; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.output.OutputFrame; -import redis.clients.jedis.Jedis; import java.util.List; import java.util.concurrent.TimeUnit; @@ -34,17 +33,17 @@ public class AbstractRedisClusterContainer { @ClassRule(order = 0) public static Network network = Network.newNetwork(); @ClassRule(order = 1) - public static GenericContainer redis1 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6371").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + public static GenericContainer redis1 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER", "6371").withNetworkMode("host").withLogConsumer(x -> log.warn("{}", ((OutputFrame) x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD", "yes").withEnv("REDIS_NODES", nodes); @ClassRule(order = 2) - public static GenericContainer redis2 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6372").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + public static GenericContainer redis2 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER", "6372").withNetworkMode("host").withLogConsumer(x -> log.warn("{}", ((OutputFrame) x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD", "yes").withEnv("REDIS_NODES", nodes); @ClassRule(order = 3) - public static GenericContainer redis3 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6373").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + public static GenericContainer redis3 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER", "6373").withNetworkMode("host").withLogConsumer(x -> log.warn("{}", ((OutputFrame) x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD", "yes").withEnv("REDIS_NODES", nodes); @ClassRule(order = 4) - public static GenericContainer redis4 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6374").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + public static GenericContainer redis4 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER", "6374").withNetworkMode("host").withLogConsumer(x -> log.warn("{}", ((OutputFrame) x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD", "yes").withEnv("REDIS_NODES", nodes); @ClassRule(order = 5) - public static GenericContainer redis5 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6375").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + public static GenericContainer redis5 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER", "6375").withNetworkMode("host").withLogConsumer(x -> log.warn("{}", ((OutputFrame) x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD", "yes").withEnv("REDIS_NODES", nodes); @ClassRule(order = 6) - public static GenericContainer redis6 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6376").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); + public static GenericContainer redis6 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER", "6376").withNetworkMode("host").withLogConsumer(x -> log.warn("{}", ((OutputFrame) x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD", "yes").withEnv("REDIS_NODES", nodes); @ClassRule(order = 100) @@ -61,12 +60,14 @@ public class AbstractRedisClusterContainer { Thread.sleep(TimeUnit.SECONDS.toMillis(5)); // otherwise not all containers have time to start String clusterCreateCommand = "echo yes | redis-cli --cluster create " + - "127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375 127.0.0.1:6376 " + + "127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375 127.0.0.1:6376 " + "--cluster-replicas 1"; log.warn("Command to init Redis Cluster: {}", clusterCreateCommand); var result = redis6.execInContainer("/bin/sh", "-c", clusterCreateCommand); log.warn("Init cluster result: {}", result); + Thread.sleep(TimeUnit.SECONDS.toMillis(5)); // otherwise cluster not always ready + log.warn("Connect to nodes: {}", nodes); System.setProperty("cache.type", "redis"); System.setProperty("redis.connection.type", "cluster"); From 358925cffe4a18822fe23f016c0bc46d1b9218d0 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 24 Jul 2024 13:35:29 +0200 Subject: [PATCH 37/42] added version for relations --- .../main/data/upgrade/3.7.0/schema_update.sql | 7 + .../server/dao/relation/RelationService.java | 2 +- .../common/data/relation/EntityRelation.java | 45 +---- .../server/dao/model/sql/RelationEntity.java | 5 +- .../dao/relation/BaseRelationService.java | 112 +++++-------- .../server/dao/relation/RelationDao.java | 24 ++- .../dao/sql/relation/JpaRelationDao.java | 157 +++++++++++------- .../relation/RelationInsertRepository.java | 2 +- .../dao/sql/relation/RelationRepository.java | 3 - .../relation/SqlRelationInsertRepository.java | 56 +++++-- .../main/resources/sql/schema-entities.sql | 3 + .../server/dao/service/AlarmServiceTest.java | 4 +- .../dao/service/RelationServiceTest.java | 92 +++++----- .../resources/sql/psql/drop-all-tables.sql | 1 + 14 files changed, 266 insertions(+), 247 deletions(-) diff --git a/application/src/main/data/upgrade/3.7.0/schema_update.sql b/application/src/main/data/upgrade/3.7.0/schema_update.sql index f964387359..ad8b76a4f3 100644 --- a/application/src/main/data/upgrade/3.7.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.7.0/schema_update.sql @@ -23,3 +23,10 @@ ALTER TABLE attribute_kv ADD COLUMN version bigint default 0; ALTER TABLE ts_kv_latest ADD COLUMN version bigint default 0; -- KV VERSIONING UPDATE END + +-- RELATION VERSIONING UPDATE START + +CREATE SEQUENCE IF NOT EXISTS relation_version_seq cache 1000; +ALTER TABLE relation ADD COLUMN version bigint default 0; + +-- RELATION VERSIONING UPDATE END diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index 05dc01a039..013b27275e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -37,7 +37,7 @@ public interface RelationService { EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - boolean saveRelation(TenantId tenantId, EntityRelation relation); + EntityRelation saveRelation(TenantId tenantId, EntityRelation relation); void saveRelations(TenantId tenantId, List relations); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 1b32cc632e..3c1bc83932 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -18,8 +18,10 @@ package org.thingsboard.server.common.data.relation; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; +import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.validation.Length; @@ -27,7 +29,8 @@ import java.io.Serializable; @Slf4j @Schema -public class EntityRelation implements Serializable { +@Data +public class EntityRelation implements HasVersion, Serializable { private static final long serialVersionUID = 2807343040519543363L; @@ -40,6 +43,7 @@ public class EntityRelation implements Serializable { @Length(fieldName = "type") private String type; private RelationTypeGroup typeGroup; + private Long version; private transient JsonNode additionalInfo; @JsonIgnore private byte[] additionalInfoBytes; @@ -70,6 +74,7 @@ public class EntityRelation implements Serializable { this.type = entityRelation.getType(); this.typeGroup = entityRelation.getTypeGroup(); this.additionalInfo = entityRelation.getAdditionalInfo(); + this.version = entityRelation.getVersion(); } @Schema(description = "JSON object with [from] Entity Id.", accessMode = Schema.AccessMode.READ_ONLY) @@ -77,35 +82,24 @@ public class EntityRelation implements Serializable { return from; } - public void setFrom(EntityId from) { - this.from = from; - } - @Schema(description = "JSON object with [to] Entity Id.", accessMode = Schema.AccessMode.READ_ONLY) public EntityId getTo() { return to; } - public void setTo(EntityId to) { - this.to = to; - } - @Schema(description = "String value of relation type.", example = "Contains") public String getType() { return type; } - public void setType(String type) { - this.type = type; - } - @Schema(description = "Represents the type group of the relation.", example = "COMMON") public RelationTypeGroup getTypeGroup() { return typeGroup; } - public void setTypeGroup(RelationTypeGroup typeGroup) { - this.typeGroup = typeGroup; + @Override + public Long getVersion() { + return version; } @Schema(description = "Additional parameters of the relation",implementation = com.fasterxml.jackson.databind.JsonNode.class) @@ -117,25 +111,4 @@ public class EntityRelation implements Serializable { BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - EntityRelation that = (EntityRelation) o; - - if (from != null ? !from.equals(that.from) : that.from != null) return false; - if (to != null ? !to.equals(that.to) : that.to != null) return false; - if (type != null ? !type.equals(that.type) : that.type != null) return false; - return typeGroup == that.typeGroup; - } - - @Override - public int hashCode() { - int result = from != null ? from.hashCode() : 0; - result = 31 * result + (to != null ? to.hashCode() : 0); - result = 31 * result + (type != null ? type.hashCode() : 0); - result = 31 * result + (typeGroup != null ? typeGroup.hashCode() : 0); - return result; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RelationEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RelationEntity.java index d7d74d4e8d..2aa5c3e573 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RelationEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RelationEntity.java @@ -23,6 +23,7 @@ import jakarta.persistence.Id; import jakarta.persistence.IdClass; import jakarta.persistence.Table; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -40,11 +41,12 @@ import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TO_TYPE_P import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TYPE_GROUP_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TYPE_PROPERTY; +@EqualsAndHashCode(callSuper = true) @Data @Entity @Table(name = RELATION_TABLE_NAME) @IdClass(RelationCompositeKey.class) -public final class RelationEntity implements ToData { +public final class RelationEntity extends VersionedEntity implements ToData { @Id @Column(name = RELATION_FROM_ID_PROPERTY, columnDefinition = "uuid") @@ -103,6 +105,7 @@ public final class RelationEntity implements ToData { } relation.setType(relationType); relation.setTypeGroup(RelationTypeGroup.valueOf(relationTypeGroup)); + relation.setVersion(version); relation.setAdditionalInfo(additionalInfo); return relation; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 5ac1d7ad3b..da383ae1bf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -21,12 +21,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; -import org.springframework.dao.ConcurrencyFailureException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; @@ -52,8 +53,6 @@ import org.thingsboard.server.dao.service.ConstraintValidator; import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.sql.relation.JpaRelationQueryExecutorService; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -149,11 +148,11 @@ public class BaseRelationService implements RelationService { return relationDao.getRelation(tenantId, from, to, relationType, typeGroup); }, RelationCacheValue::getRelation, - relations -> RelationCacheValue.builder().relation(relations).build(), false); + relation -> RelationCacheValue.builder().relation(relation).build(), false); } @Override - public boolean saveRelation(TenantId tenantId, EntityRelation relation) { + public EntityRelation saveRelation(TenantId tenantId, EntityRelation relation) { log.trace("Executing saveRelation [{}]", relation); validate(relation); var result = relationDao.saveRelation(tenantId, relation); @@ -182,11 +181,11 @@ public class BaseRelationService implements RelationService { log.trace("Executing saveRelationAsync [{}]", relation); validate(relation); var future = relationDao.saveRelationAsync(tenantId, relation); - future.addListener(() -> { + return Futures.transform(future, savedRelation -> { handleEvictEvent(EntityRelationEvent.from(relation)); eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_ADD_OR_UPDATE)); + return savedRelation != null; }, MoreExecutors.directExecutor()); - return future; } @Override @@ -194,10 +193,11 @@ public class BaseRelationService implements RelationService { log.trace("Executing DeleteRelation [{}]", relation); validate(relation); var result = relationDao.deleteRelation(tenantId, relation); - //TODO: evict cache only if the relation was deleted. Note: relationDao.deleteRelation requires improvement. - publishEvictEvent(EntityRelationEvent.from(relation)); - eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_DELETED)); - return result; + if (result != null) { + publishEvictEvent(EntityRelationEvent.from(relation)); + eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_DELETED)); + } + return result != null; } @Override @@ -205,11 +205,13 @@ public class BaseRelationService implements RelationService { log.trace("Executing deleteRelationAsync [{}]", relation); validate(relation); var future = relationDao.deleteRelationAsync(tenantId, relation); - future.addListener(() -> { - handleEvictEvent(EntityRelationEvent.from(relation)); - eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_DELETED)); + return Futures.transform(future, deletedRelation -> { + if (deletedRelation != null) { + handleEvictEvent(EntityRelationEvent.from(relation)); + eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_DELETED)); + } + return deletedRelation != null; }, MoreExecutors.directExecutor()); - return future; } @Override @@ -217,11 +219,11 @@ public class BaseRelationService implements RelationService { log.trace("Executing deleteRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); var result = relationDao.deleteRelation(tenantId, from, to, relationType, typeGroup); - //TODO: evict cache only if the relation was deleted. Note: relationDao.deleteRelation requires improvement. - EntityRelation entityRelation = new EntityRelation(from, to, relationType, typeGroup); - publishEvictEvent(EntityRelationEvent.from(entityRelation)); - eventPublisher.publishEvent(new RelationActionEvent(tenantId, entityRelation, ActionType.RELATION_DELETED)); - return result; + if (result != null) { + publishEvictEvent(EntityRelationEvent.from(result)); + eventPublisher.publishEvent(new RelationActionEvent(tenantId, result, ActionType.RELATION_DELETED)); + } + return result != null; } @Override @@ -229,9 +231,12 @@ public class BaseRelationService implements RelationService { log.trace("Executing deleteRelationAsync [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); var future = relationDao.deleteRelationAsync(tenantId, from, to, relationType, typeGroup); - EntityRelationEvent event = new EntityRelationEvent(from, to, relationType, typeGroup); - future.addListener(() -> handleEvictEvent(event), MoreExecutors.directExecutor()); - return future; + return Futures.transform(future, deletedEvent -> { + if (deletedEvent != null) { + handleEvictEvent(EntityRelationEvent.from(deletedEvent)); + } + return deletedEvent != null; + }, MoreExecutors.directExecutor()); } @Transactional @@ -250,60 +255,27 @@ public class BaseRelationService implements RelationService { public void deleteEntityRelations(TenantId tenantId, EntityId entityId, RelationTypeGroup relationTypeGroup) { log.trace("Executing deleteEntityRelations [{}]", entityId); validate(entityId); - List inboundRelations = relationTypeGroup == null - ? relationDao.findAllByTo(tenantId, entityId) - : relationDao.findAllByTo(tenantId, entityId, relationTypeGroup); - List outboundRelations = relationTypeGroup == null - ? relationDao.findAllByFrom(tenantId, entityId) - : relationDao.findAllByFrom(tenantId, entityId, relationTypeGroup); - - if (!inboundRelations.isEmpty()) { - try { - if (relationTypeGroup == null) { - relationDao.deleteInboundRelations(tenantId, entityId); - } else { - relationDao.deleteInboundRelations(tenantId, entityId, relationTypeGroup); - } - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", inboundRelations, e); - } - for (EntityRelation relation : inboundRelations) { - eventPublisher.publishEvent(EntityRelationEvent.from(relation)); - } + List inboundRelations; + if (relationTypeGroup == null) { + inboundRelations = relationDao.deleteInboundRelations(tenantId, entityId); + } else { + inboundRelations = relationDao.deleteInboundRelations(tenantId, entityId, relationTypeGroup); } - if (!outboundRelations.isEmpty()) { - if (relationTypeGroup == null) { - relationDao.deleteOutboundRelations(tenantId, entityId); - } else { - relationDao.deleteOutboundRelations(tenantId, entityId, relationTypeGroup); - } - - for (EntityRelation relation : outboundRelations) { - eventPublisher.publishEvent(EntityRelationEvent.from(relation)); - } + for (EntityRelation relation : inboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); } - } - private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, boolean deleteFromDb) { - List> results = new ArrayList<>(); - for (List relationList : relations) { - relationList.forEach(relation -> results.add(deleteAsync(tenantId, relation, deleteFromDb))); + List outboundRelations; + if (relationTypeGroup == null) { + outboundRelations = relationDao.deleteOutboundRelations(tenantId, entityId); + } else { + outboundRelations = relationDao.deleteOutboundRelations(tenantId, entityId, relationTypeGroup); } - return results; - } - private ListenableFuture deleteAsync(TenantId tenantId, EntityRelation relation, boolean deleteFromDb) { - if (deleteFromDb) { - return Futures.transform(relationDao.deleteRelationAsync(tenantId, relation), - bool -> { - handleEvictEvent(EntityRelationEvent.from(relation)); - return bool; - }, MoreExecutors.directExecutor()); - } else { - handleEvictEvent(EntityRelationEvent.from(relation)); - return Futures.immediateFuture(false); + for (EntityRelation relation : outboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index da7b17a62d..2302dd0abf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -48,29 +48,27 @@ public interface RelationDao { EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - boolean saveRelation(TenantId tenantId, EntityRelation relation); + EntityRelation saveRelation(TenantId tenantId, EntityRelation relation); - void saveRelations(TenantId tenantId, Collection relations); + List saveRelations(TenantId tenantId, List relations); - ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); + ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); - boolean deleteRelation(TenantId tenantId, EntityRelation relation); + EntityRelation deleteRelation(TenantId tenantId, EntityRelation relation); - ListenableFuture deleteRelationAsync(TenantId tenantId, EntityRelation relation); + ListenableFuture deleteRelationAsync(TenantId tenantId, EntityRelation relation); - boolean deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + EntityRelation deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - void deleteOutboundRelations(TenantId tenantId, EntityId entity); + List deleteOutboundRelations(TenantId tenantId, EntityId entity); - void deleteOutboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup); + List deleteOutboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup); - void deleteInboundRelations(TenantId tenantId, EntityId entity); + List deleteInboundRelations(TenantId tenantId, EntityId entity); - void deleteInboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup); - - ListenableFuture deleteOutboundRelationsAsync(TenantId tenantId, EntityId entity); + List deleteInboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup); List findRuleNodeToRuleChainRelations(RuleChainType ruleChainType, int limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 96cef3e909..0e726f8917 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -18,11 +18,11 @@ package org.thingsboard.server.dao.sql.relation; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.ConcurrencyFailureException; -import org.springframework.dao.DataAccessException; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -36,11 +36,19 @@ import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_ID_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.RELATION_FROM_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TO_ID_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TO_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TYPE_GROUP_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.RELATION_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN; + /** * Created by Valerii Sosliuk on 5/29/2017. */ @@ -50,6 +58,8 @@ import java.util.stream.Collectors; public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService implements RelationDao { private static final List ALL_TYPE_GROUP_NAMES = new ArrayList<>(); + private static final String RETURNING = "RETURNING from_id, from_type, to_id, to_type, relation_type, relation_type_group, nextval('relation_version_seq') as version"; + private static final String DELETE_QUERY = "DELETE FROM relation WHERE from_id = ? AND from_type = ? AND to_id = ? AND to_type = ? AND relation_type = ? AND relation_type_group = ? " + RETURNING; static { Arrays.stream(RelationTypeGroup.values()).map(RelationTypeGroup::name).forEach(ALL_TYPE_GROUP_NAMES::add); @@ -144,107 +154,138 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public boolean saveRelation(TenantId tenantId, EntityRelation relation) { - return relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null; + public EntityRelation saveRelation(TenantId tenantId, EntityRelation relation) { + return DaoUtil.getData(relationInsertRepository.saveOrUpdate(new RelationEntity(relation))); } @Override - public void saveRelations(TenantId tenantId, Collection relations) { + public List saveRelations(TenantId tenantId, List relations) { List entities = relations.stream().map(RelationEntity::new).collect(Collectors.toList()); - relationInsertRepository.saveOrUpdate(entities); + return DaoUtil.convertDataList(relationInsertRepository.saveOrUpdate(entities)); } @Override - public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { - return service.submit(() -> relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null); + public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { + return service.submit(() -> DaoUtil.getData(relationInsertRepository.saveOrUpdate(new RelationEntity(relation)))); } @Override - public boolean deleteRelation(TenantId tenantId, EntityRelation relation) { + public EntityRelation deleteRelation(TenantId tenantId, EntityRelation relation) { RelationCompositeKey key = new RelationCompositeKey(relation); return deleteRelationIfExists(key); } @Override - public ListenableFuture deleteRelationAsync(TenantId tenantId, EntityRelation relation) { + public ListenableFuture deleteRelationAsync(TenantId tenantId, EntityRelation relation) { RelationCompositeKey key = new RelationCompositeKey(relation); return service.submit( () -> deleteRelationIfExists(key)); } @Override - public boolean deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public EntityRelation deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); return deleteRelationIfExists(key); } @Override - public ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); return service.submit( () -> deleteRelationIfExists(key)); } - private boolean deleteRelationIfExists(RelationCompositeKey key) { - boolean relationExistsBeforeDelete = relationRepository.existsById(key); - if (relationExistsBeforeDelete) { - try { - relationRepository.deleteById(key); - } catch (DataAccessException e) { - log.debug("[{}] Concurrency exception while deleting relation", key, e); + private EntityRelation deleteRelationIfExists(RelationCompositeKey key) { + return jdbcTemplate.query(DELETE_QUERY, rs -> { + if (!rs.next()) { + return null; } - } - return relationExistsBeforeDelete; + EntityRelation relation = new EntityRelation(); + + var fromId = rs.getObject(RELATION_FROM_ID_PROPERTY, UUID.class); + var fromType = rs.getString(RELATION_FROM_TYPE_PROPERTY); + var toId = rs.getObject(RELATION_TO_ID_PROPERTY, UUID.class); + var toType = rs.getString(RELATION_TO_TYPE_PROPERTY); + var relationTypeGroup = rs.getString(RELATION_TYPE_GROUP_PROPERTY); + var relationType = rs.getString(RELATION_TYPE_PROPERTY); + var version = rs.getLong(VERSION_COLUMN); + + //additionalInfo ignored (no need to send extra data for delete events) + + relation.setTo(EntityIdFactory.getByTypeAndUuid(toType, toId)); + relation.setFrom(EntityIdFactory.getByTypeAndUuid(fromType, fromId)); + relation.setType(relationType); + relation.setTypeGroup(RelationTypeGroup.valueOf(relationTypeGroup)); + relation.setVersion(version); + return relation; + }, key.getFromId(), key.getFromType(), key.getToId(), key.getToType(), key.getRelationType(), key.getRelationTypeGroup()); } @Override - public void deleteOutboundRelations(TenantId tenantId, EntityId entity) { - try { - relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", entity, e); - } + public List deleteOutboundRelations(TenantId tenantId, EntityId entity) { + return deleteRelations(entity, null, false); } @Override - public void deleteOutboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup) { - try { - relationRepository.deleteByFromIdAndFromTypeAndRelationTypeGroupIn(entity.getId(), entity.getEntityType().name(), Collections.singletonList(relationTypeGroup.name())); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", entity, e); - } + public List deleteOutboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup) { + return deleteRelations(entity, Collections.singletonList(relationTypeGroup.name()), false); } @Override - public void deleteInboundRelations(TenantId tenantId, EntityId entity) { - try { - relationRepository.deleteByToIdAndToTypeAndRelationTypeGroupIn(entity.getId(), entity.getEntityType().name(), ALL_TYPE_GROUP_NAMES); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", entity, e); - } + public List deleteInboundRelations(TenantId tenantId, EntityId entity) { + return deleteRelations(entity, ALL_TYPE_GROUP_NAMES, true); } @Override - public void deleteInboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup) { - try { - relationRepository.deleteByToIdAndToTypeAndRelationTypeGroupIn(entity.getId(), entity.getEntityType().name(), Collections.singletonList(relationTypeGroup.name())); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", entity, e); - } + public List deleteInboundRelations(TenantId tenantId, EntityId entity, RelationTypeGroup relationTypeGroup) { + return deleteRelations(entity, Collections.singletonList(relationTypeGroup.name()), true); } - @Override - public ListenableFuture deleteOutboundRelationsAsync(TenantId tenantId, EntityId entity) { - return service.submit( - () -> { - boolean relationExistsBeforeDelete = relationRepository - .findAllByFromIdAndFromType(entity.getId(), entity.getEntityType().name()) - .size() > 0; - if (relationExistsBeforeDelete) { - relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); - } - return relationExistsBeforeDelete; - }); + private List deleteRelations(EntityId entityId, List relationTypeGroups, boolean inbound) { + List params = new ArrayList<>(); + params.add(entityId.getId()); + params.add(entityId.getEntityType().name()); + + StringBuilder sqlBuilder = new StringBuilder("DELETE FROM relation WHERE "); + if (inbound) { + sqlBuilder.append("to_id = ? AND to_type = ? "); + } else { + sqlBuilder.append("from_id = ? AND from_type = ? "); + } + + if (!CollectionUtils.isEmpty(relationTypeGroups)) { + sqlBuilder.append("AND relation_type_group IN (?"); + for (int i = 1; i < relationTypeGroups.size(); i++) { + sqlBuilder.append(", ?"); + } + sqlBuilder.append(")"); + params.addAll(relationTypeGroups); + } + + sqlBuilder.append(RETURNING); + + return jdbcTemplate.queryForList(sqlBuilder.toString(), params.toArray()).stream() + .map(row -> { + EntityRelation relation = new EntityRelation(); + + var fromId = row.get(RELATION_FROM_ID_PROPERTY); + var fromType = row.get(RELATION_FROM_TYPE_PROPERTY); + var toId = row.get(RELATION_TO_ID_PROPERTY); + var toType = row.get(RELATION_TO_TYPE_PROPERTY); + var relationTypeGroup = row.get(RELATION_TYPE_GROUP_PROPERTY); + var relationType = row.get(RELATION_TYPE_PROPERTY); + var version = row.get(VERSION_COLUMN); + + //additionalInfo ignored (no need to send extra data for delete events) + + relation.setTo(EntityIdFactory.getByTypeAndUuid((String) toType, (UUID) toId)); + relation.setFrom(EntityIdFactory.getByTypeAndUuid((String) fromType, (UUID) fromId)); + relation.setType((String) relationType); + relation.setTypeGroup(RelationTypeGroup.valueOf((String) relationTypeGroup)); + relation.setVersion((Long) version); + return relation; + }) + .collect(Collectors.toList()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java index 188d8afa0f..56cb5718d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java @@ -23,6 +23,6 @@ public interface RelationInsertRepository { RelationEntity saveOrUpdate(RelationEntity entity); - void saveOrUpdate(List entities); + List saveOrUpdate(List entities); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index c812788ed0..0f56a413df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -58,9 +58,6 @@ public interface RelationRepository String relationType, String relationTypeGroup); - List findAllByFromIdAndFromType(UUID fromId, - String fromType); - @Query("SELECT r FROM RelationEntity r WHERE " + "r.relationTypeGroup = 'RULE_NODE' AND r.toType = 'RULE_CHAIN' " + "AND r.toId in (SELECT id from RuleChainEntity where type = :ruleChainType )") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java index 7655ff5539..f2d374641e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java @@ -15,33 +15,39 @@ */ package org.thingsboard.server.dao.sql.relation; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.Query; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementCreator; +import org.springframework.jdbc.core.SqlProvider; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.model.sql.RelationEntity; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; -import jakarta.persistence.Query; +import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; +import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN; + @Repository @Transactional public class SqlRelationInsertRepository implements RelationInsertRepository { - private static final String INSERT_ON_CONFLICT_DO_UPDATE_JPA = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + - " VALUES (:fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) " + - "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = :additionalInfo returning *"; - - private static final String INSERT_ON_CONFLICT_DO_UPDATE_JDBC = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + - " VALUES (?, ?, ?, ?, ?, ?, ?) " + - "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = ?"; + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JPA = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, version, additional_info)" + + " VALUES (:fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, nextval('relation_version_seq'), :additionalInfo) " + + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = :additionalInfo, version = nextval('relation_version_seq') returning *"; + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JDBC = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, version, additional_info)" + + " VALUES (?, ?, ?, ?, ?, ?, nextval('relation_version_seq'), ?) " + + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = ?, version = nextval('relation_version_seq')"; @PersistenceContext protected EntityManager entityManager; @@ -71,8 +77,9 @@ public class SqlRelationInsertRepository implements RelationInsertRepository { } @Override - public void saveOrUpdate(List entities) { - jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { + public List saveOrUpdate(List entities) { + KeyHolder keyHolder = new GeneratedKeyHolder(); + jdbcTemplate.batchUpdate(new SequencePreparedStatementCreator(INSERT_ON_CONFLICT_DO_UPDATE_JDBC), new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { RelationEntity relation = entities.get(i); @@ -98,7 +105,30 @@ public class SqlRelationInsertRepository implements RelationInsertRepository { public int getBatchSize() { return entities.size(); } - }); + }, keyHolder); + + var seqNumbers = keyHolder.getKeyList(); + + for (int i = 0; i < entities.size(); i++) { + entities.get(i).setVersion((Long) seqNumbers.get(i).get(VERSION_COLUMN)); + } + + return entities; + } + + private record SequencePreparedStatementCreator(String sql) implements PreparedStatementCreator, SqlProvider { + + private static final String[] COLUMNS = {VERSION_COLUMN}; + + @Override + public PreparedStatement createPreparedStatement(Connection con) throws SQLException { + return con.prepareStatement(sql, COLUMNS); + } + + @Override + public String getSql() { + return this.sql; + } } } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 9e6ab22248..28b842076a 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -420,6 +420,8 @@ CREATE TABLE IF NOT EXISTS error_event ( e_error varchar ) PARTITION BY RANGE (ts); +CREATE SEQUENCE IF NOT EXISTS relation_version_seq cache 1000; + CREATE TABLE IF NOT EXISTS relation ( from_id uuid, from_type varchar(255), @@ -428,6 +430,7 @@ CREATE TABLE IF NOT EXISTS relation ( relation_type_group varchar(255), relation_type varchar(255), additional_info varchar, + version bigint default 0, CONSTRAINT relation_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type) ); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java index 6c3a359f73..aa1c96d84b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java @@ -340,7 +340,7 @@ public class AlarmServiceTest extends AbstractServiceTest { EntityRelation relation = new EntityRelation(parentId, childId, EntityRelation.CONTAINS_TYPE); - Assert.assertTrue(relationService.saveRelation(tenantId, relation)); + Assert.assertNotNull(relationService.saveRelation(tenantId, relation)); long ts = System.currentTimeMillis(); AlarmApiCallResult result = alarmService.createAlarm(AlarmCreateOrUpdateActiveRequest.builder() @@ -877,7 +877,7 @@ public class AlarmServiceTest extends AbstractServiceTest { EntityRelation relation = new EntityRelation(parentId, childId, EntityRelation.CONTAINS_TYPE); - Assert.assertTrue(relationService.saveRelation(tenantId, relation)); + Assert.assertNotNull(relationService.saveRelation(tenantId, relation)); long ts = System.currentTimeMillis(); AlarmApiCallResult result = alarmService.createAlarm(AlarmCreateOrUpdateActiveRequest.builder() diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java index 4e20744ef7..710a69fac9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/RelationServiceTest.java @@ -57,13 +57,13 @@ public class RelationServiceTest extends AbstractServiceTest { } @Test - public void testSaveRelation() throws ExecutionException, InterruptedException { + public void testSaveRelation() { AssetId parentId = new AssetId(Uuids.timeBased()); AssetId childId = new AssetId(Uuids.timeBased()); EntityRelation relation = new EntityRelation(parentId, childId, EntityRelation.CONTAINS_TYPE); - Assert.assertTrue(saveRelation(relation)); + Assert.assertNotNull(saveRelation(relation)); Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); @@ -204,8 +204,8 @@ public class RelationServiceTest extends AbstractServiceTest { Assert.assertEquals(0, relations.size()); } - private Boolean saveRelation(EntityRelation relationA1) { - return relationService.saveRelation(SYSTEM_TENANT_ID, relationA1); + private EntityRelation saveRelation(EntityRelation relation) { + return relationService.saveRelation(SYSTEM_TENANT_ID, relation); } @Test @@ -265,9 +265,9 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationB = new EntityRelation(assetB, assetC, EntityRelation.CONTAINS_TYPE); EntityRelation relationC = new EntityRelation(assetC, assetA, EntityRelation.CONTAINS_TYPE); - saveRelation(relationA); - saveRelation(relationB); - saveRelation(relationC); + relationA = saveRelation(relationA); + relationB = saveRelation(relationB); + relationC = saveRelation(relationC); EntityRelationsQuery query = new EntityRelationsQuery(); query.setParameters(new RelationsSearchParameters(assetA, EntitySearchDirection.FROM, -1, false)); @@ -299,8 +299,8 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationBD = new EntityRelation(assetB, deviceD, EntityRelation.CONTAINS_TYPE); - saveRelation(relationAB); - saveRelation(relationBC); + relationAB = saveRelation(relationAB); + relationBC = saveRelation(relationBC); saveRelation(relationBD); EntityRelationsQuery query = new EntityRelationsQuery(); @@ -329,26 +329,20 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationAB = new EntityRelation(root, left, EntityRelation.CONTAINS_TYPE); EntityRelation relationBC = new EntityRelation(root, right, EntityRelation.CONTAINS_TYPE); - saveRelation(relationAB); - expected.add(relationAB); - - saveRelation(relationBC); - expected.add(relationBC); + expected.add(saveRelation(relationAB)); + expected.add(saveRelation(relationBC)); for (int i = 0; i < maxLevel; i++) { var newLeft = new AssetId(Uuids.timeBased()); var newRight = new AssetId(Uuids.timeBased()); EntityRelation relationLeft = new EntityRelation(left, newLeft, EntityRelation.CONTAINS_TYPE); EntityRelation relationRight = new EntityRelation(right, newRight, EntityRelation.CONTAINS_TYPE); - saveRelation(relationLeft); - expected.add(relationLeft); - saveRelation(relationRight); - expected.add(relationRight); + expected.add(saveRelation(relationLeft)); + expected.add(saveRelation(relationRight)); left = newLeft; right = newRight; } - EntityRelationsQuery query = new EntityRelationsQuery(); query.setParameters(new RelationsSearchParameters(root, EntitySearchDirection.FROM, -1, false)); query.setFilters(Collections.singletonList(new RelationEntityTypeFilter(EntityRelation.CONTAINS_TYPE, Collections.singletonList(EntityType.ASSET)))); @@ -372,7 +366,7 @@ public class RelationServiceTest extends AbstractServiceTest { relation.setTo(new AssetId(Uuids.timeBased())); relation.setType(EntityRelation.CONTAINS_TYPE); Assertions.assertThrows(DataValidationException.class, () -> { - Assert.assertTrue(saveRelation(relation)); + Assert.assertNotNull(saveRelation(relation)); }); } @@ -382,7 +376,7 @@ public class RelationServiceTest extends AbstractServiceTest { relation.setFrom(new AssetId(Uuids.timeBased())); relation.setType(EntityRelation.CONTAINS_TYPE); Assertions.assertThrows(DataValidationException.class, () -> { - Assert.assertTrue(saveRelation(relation)); + Assert.assertNotNull(saveRelation(relation)); }); } @@ -392,7 +386,7 @@ public class RelationServiceTest extends AbstractServiceTest { relation.setFrom(new AssetId(Uuids.timeBased())); relation.setTo(new AssetId(Uuids.timeBased())); Assertions.assertThrows(DataValidationException.class, () -> { - Assert.assertTrue(saveRelation(relation)); + Assert.assertNotNull(saveRelation(relation)); }); } @@ -414,10 +408,10 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationC = new EntityRelation(assetC, assetD, EntityRelation.CONTAINS_TYPE); EntityRelation relationD = new EntityRelation(assetC, assetE, EntityRelation.CONTAINS_TYPE); - saveRelation(relationA); - saveRelation(relationB); - saveRelation(relationC); - saveRelation(relationD); + relationA = saveRelation(relationA); + relationB = saveRelation(relationB); + relationC = saveRelation(relationC); + relationD = saveRelation(relationD); EntityRelationsQuery query = new EntityRelationsQuery(); query.setParameters(new RelationsSearchParameters(assetA, EntitySearchDirection.FROM, -1, true)); @@ -450,9 +444,9 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationB = new EntityRelation(assetB, assetC, EntityRelation.CONTAINS_TYPE); EntityRelation relationC = new EntityRelation(assetC, assetD, EntityRelation.CONTAINS_TYPE); - saveRelation(relationA); - saveRelation(relationB); - saveRelation(relationC); + relationA = saveRelation(relationA); + relationB = saveRelation(relationB); + relationC = saveRelation(relationC); EntityRelationsQuery query = new EntityRelationsQuery(); query.setParameters(new RelationsSearchParameters(assetA, EntitySearchDirection.FROM, -1, true)); @@ -494,12 +488,12 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationE = new EntityRelation(assetD, assetF, EntityRelation.CONTAINS_TYPE); EntityRelation relationF = new EntityRelation(assetD, assetG, EntityRelation.CONTAINS_TYPE); - saveRelation(relationA); - saveRelation(relationB); - saveRelation(relationC); - saveRelation(relationD); - saveRelation(relationE); - saveRelation(relationF); + relationA = saveRelation(relationA); + relationB = saveRelation(relationB); + relationC = saveRelation(relationC); + relationD = saveRelation(relationD); + relationE = saveRelation(relationE); + relationF = saveRelation(relationF); EntityRelationsQuery query = new EntityRelationsQuery(); query.setParameters(new RelationsSearchParameters(assetA, EntitySearchDirection.FROM, 2, true)); @@ -547,12 +541,12 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationE = new EntityRelation(assetD, assetF, EntityRelation.CONTAINS_TYPE); EntityRelation relationF = new EntityRelation(assetD, assetG, EntityRelation.CONTAINS_TYPE); - saveRelation(relationA); - saveRelation(relationB); - saveRelation(relationC); - saveRelation(relationD); - saveRelation(relationE); - saveRelation(relationF); + relationA = saveRelation(relationA); + relationB = saveRelation(relationB); + relationC = saveRelation(relationC); + relationD = saveRelation(relationD); + relationE = saveRelation(relationE); + relationF = saveRelation(relationF); EntityRelationsQuery query = new EntityRelationsQuery(); query.setParameters(new RelationsSearchParameters(assetA, EntitySearchDirection.FROM, 2, false)); @@ -600,12 +594,12 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation relationE = new EntityRelation(assetD, assetF, EntityRelation.CONTAINS_TYPE); EntityRelation relationF = new EntityRelation(assetD, assetG, EntityRelation.CONTAINS_TYPE); - saveRelation(relationA); - saveRelation(relationB); - saveRelation(relationC); - saveRelation(relationD); - saveRelation(relationE); - saveRelation(relationF); + relationA = saveRelation(relationA); + relationB = saveRelation(relationB); + relationC = saveRelation(relationC); + relationD = saveRelation(relationD); + relationE = saveRelation(relationE); + relationF = saveRelation(relationF); EntityRelationsQuery query = new EntityRelationsQuery(); query.setParameters(new RelationsSearchParameters(assetA, EntitySearchDirection.FROM, -1, false)); @@ -670,8 +664,8 @@ public class RelationServiceTest extends AbstractServiceTest { EntityRelation firstRelation = new EntityRelation(rootAsset, firstAsset, EntityRelation.CONTAINS_TYPE); EntityRelation secondRelation = new EntityRelation(rootAsset, secondAsset, EntityRelation.CONTAINS_TYPE); - saveRelation(firstRelation); - saveRelation(secondRelation); + firstRelation = saveRelation(firstRelation); + secondRelation = saveRelation(secondRelation); if (!lastLvlOnly || lvl == 1) { entityRelations.add(firstRelation); diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index e1c35d7a21..da6eca161b 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -34,6 +34,7 @@ DROP TABLE IF EXISTS stats_event; DROP TABLE IF EXISTS lc_event; DROP TABLE IF EXISTS error_event; DROP TABLE IF EXISTS relation; +DROP SEQUENCE IF EXISTS relation_version_seq; DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS ts_kv; DROP TABLE IF EXISTS ts_kv_latest; From a15ec624d769284bf6340055d96bbae9ccc72e9d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 24 Jul 2024 16:52:35 +0200 Subject: [PATCH 38/42] fixed RelationCacheTest --- .../thingsboard/server/dao/service/RelationCacheTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/RelationCacheTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/RelationCacheTest.java index 495f3c24e0..e86517930f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/RelationCacheTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/RelationCacheTest.java @@ -85,8 +85,12 @@ public class RelationCacheTest extends AbstractServiceTest { @Test public void testDeleteRelations_EvictsCache() { + EntityRelation relation = new EntityRelation(ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE); when(relationDao.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON)) - .thenReturn(new EntityRelation(ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE)); + .thenReturn(relation); + + when(relationDao.deleteRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON)) + .thenReturn(relation); relationService.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); relationService.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); From 977da8bc8a72e952d69c825e120446c0da516018 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 25 Jul 2024 12:01:42 +0200 Subject: [PATCH 39/42] fixed tests --- .../controller/EntityRelationController.java | 8 +++--- .../DefaultTbEntityRelationService.java | 16 ++++++----- .../relation/TbEntityRelationService.java | 4 +-- .../EntityRelationControllerTest.java | 14 +++++----- .../server/edge/RelationEdgeTest.java | 11 ++++---- .../service/sync/vc/VersionControlTest.java | 3 +-- .../server/dao/relation/RelationService.java | 2 +- .../common/data/relation/EntityRelation.java | 19 ++++++++----- .../dao/relation/BaseRelationService.java | 27 ++++++++++--------- 9 files changed, 56 insertions(+), 48 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index 0cd31a729b..0a471e3a2d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -81,7 +81,7 @@ public class EntityRelationController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/relation", method = RequestMethod.POST) @ResponseStatus(value = HttpStatus.OK) - public void saveRelation(@Parameter(description = "A JSON value representing the relation.", required = true) + public EntityRelation saveRelation(@Parameter(description = "A JSON value representing the relation.", required = true) @RequestBody EntityRelation relation) throws ThingsboardException { checkNotNull(relation); checkCanCreateRelation(relation.getFrom()); @@ -90,7 +90,7 @@ public class EntityRelationController extends BaseController { relation.setTypeGroup(RelationTypeGroup.COMMON); } - tbEntityRelationService.save(getTenantId(), getCurrentUser().getCustomerId(), relation, getCurrentUser()); + return tbEntityRelationService.save(getTenantId(), getCurrentUser().getCustomerId(), relation, getCurrentUser()); } @ApiOperation(value = "Delete Relation (deleteRelation)", @@ -98,7 +98,7 @@ public class EntityRelationController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/relation", method = RequestMethod.DELETE, params = {FROM_ID, FROM_TYPE, RELATION_TYPE, TO_ID, TO_TYPE}) @ResponseStatus(value = HttpStatus.OK) - public void deleteRelation(@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_ID) String strFromId, + public EntityRelation deleteRelation(@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_ID) String strFromId, @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_TYPE) String strFromType, @Parameter(description = RELATION_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(RELATION_TYPE) String strRelationType, @Parameter(description = RELATION_TYPE_GROUP_PARAM_DESCRIPTION) @RequestParam(value = "relationTypeGroup", required = false) String strRelationTypeGroup, @@ -116,7 +116,7 @@ public class EntityRelationController extends BaseController { RelationTypeGroup relationTypeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); EntityRelation relation = new EntityRelation(fromId, toId, strRelationType, relationTypeGroup); - tbEntityRelationService.delete(getTenantId(), getCurrentUser().getCustomerId(), relation, getCurrentUser()); + return tbEntityRelationService.delete(getTenantId(), getCurrentUser().getCustomerId(), relation, getCurrentUser()); } @ApiOperation(value = "Delete common relations (deleteCommonRelations)", diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/DefaultTbEntityRelationService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/DefaultTbEntityRelationService.java index 724c077e62..e9cfc109f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/DefaultTbEntityRelationService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/DefaultTbEntityRelationService.java @@ -39,12 +39,13 @@ public class DefaultTbEntityRelationService extends AbstractTbEntityService impl private final RelationService relationService; @Override - public void save(TenantId tenantId, CustomerId customerId, EntityRelation relation, User user) throws ThingsboardException { + public EntityRelation save(TenantId tenantId, CustomerId customerId, EntityRelation relation, User user) throws ThingsboardException { ActionType actionType = ActionType.RELATION_ADD_OR_UPDATE; try { - relationService.saveRelation(tenantId, relation); + var savedRelation = relationService.saveRelation(tenantId, relation); logEntityActionService.logEntityRelationAction(tenantId, customerId, - relation, user, actionType, null, relation); + savedRelation, user, actionType, null, savedRelation); + return savedRelation; } catch (Exception e) { logEntityActionService.logEntityRelationAction(tenantId, customerId, relation, user, actionType, e, relation); @@ -53,14 +54,15 @@ public class DefaultTbEntityRelationService extends AbstractTbEntityService impl } @Override - public void delete(TenantId tenantId, CustomerId customerId, EntityRelation relation, User user) throws ThingsboardException { + public EntityRelation delete(TenantId tenantId, CustomerId customerId, EntityRelation relation, User user) throws ThingsboardException { ActionType actionType = ActionType.RELATION_DELETED; try { - boolean found = relationService.deleteRelation(tenantId, relation.getFrom(), relation.getTo(), relation.getType(), relation.getTypeGroup()); - if (!found) { + var found = relationService.deleteRelation(tenantId, relation.getFrom(), relation.getTo(), relation.getType(), relation.getTypeGroup()); + if (found == null) { throw new ThingsboardException("Requested item wasn't found!", ThingsboardErrorCode.ITEM_NOT_FOUND); } - logEntityActionService.logEntityRelationAction(tenantId, customerId, relation, user, actionType, null, relation); + logEntityActionService.logEntityRelationAction(tenantId, customerId, found, user, actionType, null, found); + return found; } catch (Exception e) { logEntityActionService.logEntityRelationAction(tenantId, customerId, relation, user, actionType, e, relation); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/TbEntityRelationService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/TbEntityRelationService.java index 7b732ff9ee..0ef75d0354 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/TbEntityRelationService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entity/relation/TbEntityRelationService.java @@ -24,9 +24,9 @@ import org.thingsboard.server.common.data.relation.EntityRelation; public interface TbEntityRelationService { - void save(TenantId tenantId, CustomerId customerId, EntityRelation entity, User user) throws ThingsboardException; + EntityRelation save(TenantId tenantId, CustomerId customerId, EntityRelation entity, User user) throws ThingsboardException; - void delete(TenantId tenantId, CustomerId customerId, EntityRelation entity, User user) throws ThingsboardException; + EntityRelation delete(TenantId tenantId, CustomerId customerId, EntityRelation entity, User user) throws ThingsboardException; void deleteCommonRelations(TenantId tenantId, CustomerId customerId, EntityId entityId, User user) throws ThingsboardException; diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java index 48e0fb6196..f2464f3e47 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java @@ -103,7 +103,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - doPost("/api/relation", relation).andExpect(status().isOk()); + relation = doPost("/api/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", mainDevice.getUuidId(), EntityType.DEVICE, @@ -315,7 +315,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Device device = buildSimpleDevice("Test device 1"); EntityRelation relation = createFromRelation(mainDevice, device, "CONTAINS"); - doPost("/api/relation", relation).andExpect(status().isOk()); + relation = doPost("/api/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", mainDevice.getUuidId(), EntityType.DEVICE, @@ -329,11 +329,11 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - doDelete(url).andExpect(status().isOk()); + var deletedRelation = doDelete(url, EntityRelation.class); - testNotifyEntityAllOneTimeRelation(foundRelation, + testNotifyEntityAllOneTimeRelation(deletedRelation, savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.RELATION_DELETED, foundRelation); + ActionType.RELATION_DELETED, deletedRelation); doGet(url).andExpect(status().is4xxClientError()); } @@ -523,7 +523,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { @Test public void testCreateRelationFromTenantToDevice() throws Exception { EntityRelation relation = new EntityRelation(tenantAdmin.getTenantId(), mainDevice.getId(), "CONTAINS"); - doPost("/api/relation", relation).andExpect(status().isOk()); + relation = doPost("/api/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", tenantAdmin.getTenantId(), EntityType.TENANT, @@ -539,7 +539,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { @Test public void testCreateRelationFromDeviceToTenant() throws Exception { EntityRelation relation = new EntityRelation(mainDevice.getId(), tenantAdmin.getTenantId(), "CONTAINS"); - doPost("/api/relation", relation).andExpect(status().isOk()); + relation = doPost("/api/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", mainDevice.getUuidId(), EntityType.DEVICE, diff --git a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java index d89b601866..baeafd3ad2 100644 --- a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java @@ -48,7 +48,7 @@ public class RelationEdgeTest extends AbstractEdgeTest { relation.setTo(asset.getId()); relation.setTypeGroup(RelationTypeGroup.COMMON); edgeImitator.expectMessageAmount(1); - doPost("/api/relation", relation); + relation = doPost("/api/relation", relation, EntityRelation.class); Assert.assertTrue(edgeImitator.waitForMessages()); AbstractMessage latestMessage = edgeImitator.getLatestMessage(); Assert.assertTrue(latestMessage instanceof RelationUpdateMsg); @@ -60,21 +60,20 @@ public class RelationEdgeTest extends AbstractEdgeTest { // delete relation edgeImitator.expectMessageAmount(1); - doDelete("/api/relation?" + + var deletedRelation = doDelete("/api/relation?" + "fromId=" + relation.getFrom().getId().toString() + "&fromType=" + relation.getFrom().getEntityType().name() + "&relationType=" + relation.getType() + "&relationTypeGroup=" + relation.getTypeGroup().name() + "&toId=" + relation.getTo().getId().toString() + - "&toType=" + relation.getTo().getEntityType().name()) - .andExpect(status().isOk()); + "&toType=" + relation.getTo().getEntityType().name(), EntityRelation.class); Assert.assertTrue(edgeImitator.waitForMessages()); latestMessage = edgeImitator.getLatestMessage(); Assert.assertTrue(latestMessage instanceof RelationUpdateMsg); relationUpdateMsg = (RelationUpdateMsg) latestMessage; entityRelation = JacksonUtil.fromString(relationUpdateMsg.getEntity(), EntityRelation.class, true); Assert.assertNotNull(entityRelation); - Assert.assertEquals(relation, entityRelation); + Assert.assertEquals(deletedRelation, entityRelation); Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, relationUpdateMsg.getMsgType()); } @@ -119,7 +118,7 @@ public class RelationEdgeTest extends AbstractEdgeTest { deviceToAssetRelation.setTypeGroup(RelationTypeGroup.COMMON); edgeImitator.expectMessageAmount(1); - doPost("/api/relation", deviceToAssetRelation); + deviceToAssetRelation = doPost("/api/relation", deviceToAssetRelation, EntityRelation.class); Assert.assertTrue(edgeImitator.waitForMessages()); EntityRelation assetToTenantRelation = new EntityRelation(); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index 8b8f89fabf..a55bcc849d 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -937,8 +937,7 @@ public class VersionControlTest extends AbstractControllerTest { relation.setType(EntityRelation.MANAGES_TYPE); relation.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); relation.setTypeGroup(RelationTypeGroup.COMMON); - doPost("/api/relation", relation).andExpect(status().isOk()); - return relation; + return doPost("/api/relation", relation, EntityRelation.class); } protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index 013b27275e..fbc79719e0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -47,7 +47,7 @@ public interface RelationService { ListenableFuture deleteRelationAsync(TenantId tenantId, EntityRelation relation); - boolean deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + EntityRelation deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 3c1bc83932..62b5caafd3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -18,7 +18,10 @@ package org.thingsboard.server.common.data.relation; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasVersion; @@ -29,7 +32,8 @@ import java.io.Serializable; @Slf4j @Schema -@Data +@EqualsAndHashCode(exclude = "additionalInfoBytes") +@ToString(exclude = {"additionalInfoBytes"}) public class EntityRelation implements HasVersion, Serializable { private static final long serialVersionUID = 2807343040519543363L; @@ -38,11 +42,17 @@ public class EntityRelation implements HasVersion, Serializable { public static final String CONTAINS_TYPE = "Contains"; public static final String MANAGES_TYPE = "Manages"; + @Setter private EntityId from; + @Setter private EntityId to; + @Setter @Length(fieldName = "type") private String type; + @Setter private RelationTypeGroup typeGroup; + @Getter + @Setter private Long version; private transient JsonNode additionalInfo; @JsonIgnore @@ -97,11 +107,6 @@ public class EntityRelation implements HasVersion, Serializable { return typeGroup; } - @Override - public Long getVersion() { - return version; - } - @Schema(description = "Additional parameters of the relation",implementation = com.fasterxml.jackson.databind.JsonNode.class) public JsonNode getAdditionalInfo() { return BaseDataWithAdditionalInfo.getJson(() -> additionalInfo, () -> additionalInfoBytes); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index da383ae1bf..0ee34f2b9c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -156,8 +156,8 @@ public class BaseRelationService implements RelationService { log.trace("Executing saveRelation [{}]", relation); validate(relation); var result = relationDao.saveRelation(tenantId, relation); - publishEvictEvent(EntityRelationEvent.from(relation)); - eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_ADD_OR_UPDATE)); + publishEvictEvent(EntityRelationEvent.from(result)); + eventPublisher.publishEvent(new RelationActionEvent(tenantId, result, ActionType.RELATION_ADD_OR_UPDATE)); return result; } @@ -167,10 +167,11 @@ public class BaseRelationService implements RelationService { for (EntityRelation relation : relations) { validate(relation); } + List savedRelations = new ArrayList<>(relations.size()); for (List partition : Lists.partition(relations, 1024)) { - relationDao.saveRelations(tenantId, partition); + savedRelations.addAll(relationDao.saveRelations(tenantId, partition)); } - for (EntityRelation relation : relations) { + for (EntityRelation relation : savedRelations) { publishEvictEvent(EntityRelationEvent.from(relation)); eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_ADD_OR_UPDATE)); } @@ -182,8 +183,10 @@ public class BaseRelationService implements RelationService { validate(relation); var future = relationDao.saveRelationAsync(tenantId, relation); return Futures.transform(future, savedRelation -> { - handleEvictEvent(EntityRelationEvent.from(relation)); - eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_ADD_OR_UPDATE)); + if (savedRelation != null) { + handleEvictEvent(EntityRelationEvent.from(savedRelation)); + eventPublisher.publishEvent(new RelationActionEvent(tenantId, savedRelation, ActionType.RELATION_ADD_OR_UPDATE)); + } return savedRelation != null; }, MoreExecutors.directExecutor()); } @@ -194,8 +197,8 @@ public class BaseRelationService implements RelationService { validate(relation); var result = relationDao.deleteRelation(tenantId, relation); if (result != null) { - publishEvictEvent(EntityRelationEvent.from(relation)); - eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_DELETED)); + publishEvictEvent(EntityRelationEvent.from(result)); + eventPublisher.publishEvent(new RelationActionEvent(tenantId, result, ActionType.RELATION_DELETED)); } return result != null; } @@ -207,15 +210,15 @@ public class BaseRelationService implements RelationService { var future = relationDao.deleteRelationAsync(tenantId, relation); return Futures.transform(future, deletedRelation -> { if (deletedRelation != null) { - handleEvictEvent(EntityRelationEvent.from(relation)); - eventPublisher.publishEvent(new RelationActionEvent(tenantId, relation, ActionType.RELATION_DELETED)); + handleEvictEvent(EntityRelationEvent.from(deletedRelation)); + eventPublisher.publishEvent(new RelationActionEvent(tenantId, deletedRelation, ActionType.RELATION_DELETED)); } return deletedRelation != null; }, MoreExecutors.directExecutor()); } @Override - public boolean deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public EntityRelation deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { log.trace("Executing deleteRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); var result = relationDao.deleteRelation(tenantId, from, to, relationType, typeGroup); @@ -223,7 +226,7 @@ public class BaseRelationService implements RelationService { publishEvictEvent(EntityRelationEvent.from(result)); eventPublisher.publishEvent(new RelationActionEvent(tenantId, result, ActionType.RELATION_DELETED)); } - return result != null; + return result; } @Override From 922847baf6fac4a00dbbf0d53ec8e4407e249973 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 26 Jul 2024 14:03:49 +0200 Subject: [PATCH 40/42] minor refactoring --- .../dao/model/sql/AttributeKvEntity.java | 18 ++++++----- .../server/dao/model/sql/RelationEntity.java | 8 +++-- .../server/dao/model/sql/VersionedEntity.java | 30 ------------------- 3 files changed, 15 insertions(+), 41 deletions(-) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/VersionedEntity.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java index d7bd23db4d..615932ddde 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AttributeKvEntity.java @@ -15,8 +15,12 @@ */ package org.thingsboard.server.dao.model.sql; +import jakarta.persistence.Column; +import jakarta.persistence.EmbeddedId; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import jakarta.persistence.Transient; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; @@ -27,11 +31,6 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.dao.model.ToData; -import jakarta.persistence.Column; -import jakarta.persistence.EmbeddedId; -import jakarta.persistence.Entity; -import jakarta.persistence.Table; -import jakarta.persistence.Transient; import java.io.Serializable; import static org.thingsboard.server.dao.model.ModelConstants.BOOLEAN_VALUE_COLUMN; @@ -40,12 +39,12 @@ import static org.thingsboard.server.dao.model.ModelConstants.JSON_VALUE_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.LAST_UPDATE_TS_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.LONG_VALUE_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.STRING_VALUE_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN; @Data -@EqualsAndHashCode(callSuper = true) @Entity @Table(name = "attribute_kv") -public class AttributeKvEntity extends VersionedEntity implements ToData, Serializable { +public class AttributeKvEntity implements ToData, Serializable { @EmbeddedId private AttributeKvCompositeKey id; @@ -68,6 +67,9 @@ public class AttributeKvEntity extends VersionedEntity implements ToData { +public final class RelationEntity implements ToData { @Id @Column(name = RELATION_FROM_ID_PROPERTY, columnDefinition = "uuid") @@ -72,6 +71,9 @@ public final class RelationEntity extends VersionedEntity implements ToData Date: Mon, 29 Jul 2024 21:31:14 +0200 Subject: [PATCH 41/42] set version_seq cache to 1 --- application/src/main/data/upgrade/3.7.0/schema_update.sql | 6 +++--- dao/src/main/resources/sql/schema-entities.sql | 6 +++--- dao/src/main/resources/sql/schema-timescale.sql | 2 +- dao/src/main/resources/sql/schema-ts-latest-psql.sql | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/application/src/main/data/upgrade/3.7.0/schema_update.sql b/application/src/main/data/upgrade/3.7.0/schema_update.sql index ad8b76a4f3..a52eb73783 100644 --- a/application/src/main/data/upgrade/3.7.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.7.0/schema_update.sql @@ -16,8 +16,8 @@ -- KV VERSIONING UPDATE START -CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1000; -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1; +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1; ALTER TABLE attribute_kv ADD COLUMN version bigint default 0; ALTER TABLE ts_kv_latest ADD COLUMN version bigint default 0; @@ -26,7 +26,7 @@ ALTER TABLE ts_kv_latest ADD COLUMN version bigint default 0; -- RELATION VERSIONING UPDATE START -CREATE SEQUENCE IF NOT EXISTS relation_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS relation_version_seq cache 1; ALTER TABLE relation ADD COLUMN version bigint default 0; -- RELATION VERSIONING UPDATE END diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 28b842076a..391aaf9e5f 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -102,7 +102,7 @@ CREATE TABLE IF NOT EXISTS audit_log ( action_failure_details varchar(1000000) ) PARTITION BY RANGE (created_time); -CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS attribute_kv_version_seq cache 1; CREATE TABLE IF NOT EXISTS attribute_kv ( entity_id uuid, @@ -420,7 +420,7 @@ CREATE TABLE IF NOT EXISTS error_event ( e_error varchar ) PARTITION BY RANGE (ts); -CREATE SEQUENCE IF NOT EXISTS relation_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS relation_version_seq cache 1; CREATE TABLE IF NOT EXISTS relation ( from_id uuid, @@ -544,7 +544,7 @@ CREATE TABLE IF NOT EXISTS entity_view ( CONSTRAINT entity_view_external_id_unq_key UNIQUE (tenant_id, external_id) ); -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1; CREATE TABLE IF NOT EXISTS ts_kv_latest ( diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index 1d5b67a95e..6142770c1c 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS key_dictionary ( CONSTRAINT key_dictionary_id_pkey PRIMARY KEY (key) ); -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1; CREATE TABLE IF NOT EXISTS ts_kv_latest ( entity_id uuid NOT NULL, diff --git a/dao/src/main/resources/sql/schema-ts-latest-psql.sql b/dao/src/main/resources/sql/schema-ts-latest-psql.sql index 4892e40176..725c14e9bf 100644 --- a/dao/src/main/resources/sql/schema-ts-latest-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-latest-psql.sql @@ -14,7 +14,7 @@ -- limitations under the License. -- -CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1000; +CREATE SEQUENCE IF NOT EXISTS ts_kv_latest_version_seq cache 1; CREATE TABLE IF NOT EXISTS ts_kv_latest ( From a1e8ec4c02fd1355ece7526b6bdf034612f17409 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 2 Aug 2024 11:06:29 +0200 Subject: [PATCH 42/42] created separate api for relations with returning entity --- .../controller/EntityRelationController.java | 40 +++++++++++++++++-- .../EntityRelationControllerTest.java | 14 ++++--- .../server/edge/RelationEdgeTest.java | 6 +-- .../service/sync/vc/VersionControlTest.java | 2 +- .../thingsboard/rest/client/RestClient.java | 24 +++++++++++ 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index 0a471e3a2d..37015de22a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -81,8 +81,24 @@ public class EntityRelationController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/relation", method = RequestMethod.POST) @ResponseStatus(value = HttpStatus.OK) - public EntityRelation saveRelation(@Parameter(description = "A JSON value representing the relation.", required = true) - @RequestBody EntityRelation relation) throws ThingsboardException { + public void saveRelation(@Parameter(description = "A JSON value representing the relation.", required = true) + @RequestBody EntityRelation relation) throws ThingsboardException { + doSave(relation); + } + + @ApiOperation(value = "Create Relation (saveRelationV2)", + notes = "Creates or updates a relation between two entities in the platform. " + + "Relations unique key is a combination of from/to entity id and relation type group and relation type. " + + SECURITY_CHECKS_ENTITIES_DESCRIPTION) + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/v2/relation", method = RequestMethod.POST) + @ResponseStatus(value = HttpStatus.OK) + public EntityRelation saveRelationV2(@Parameter(description = "A JSON value representing the relation.", required = true) + @RequestBody EntityRelation relation) throws ThingsboardException { + return doSave(relation); + } + + private EntityRelation doSave(EntityRelation relation) throws ThingsboardException { checkNotNull(relation); checkCanCreateRelation(relation.getFrom()); checkCanCreateRelation(relation.getTo()); @@ -98,12 +114,30 @@ public class EntityRelationController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/relation", method = RequestMethod.DELETE, params = {FROM_ID, FROM_TYPE, RELATION_TYPE, TO_ID, TO_TYPE}) @ResponseStatus(value = HttpStatus.OK) - public EntityRelation deleteRelation(@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_ID) String strFromId, + public void deleteRelation(@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_ID) String strFromId, @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_TYPE) String strFromType, @Parameter(description = RELATION_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(RELATION_TYPE) String strRelationType, @Parameter(description = RELATION_TYPE_GROUP_PARAM_DESCRIPTION) @RequestParam(value = "relationTypeGroup", required = false) String strRelationTypeGroup, @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @RequestParam(TO_ID) String strToId, @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(TO_TYPE) String strToType) throws ThingsboardException { + doDelete(strFromId, strFromType, strRelationType, strRelationTypeGroup, strToId, strToType); + } + + @ApiOperation(value = "Delete Relation (deleteRelationV2)", + notes = "Deletes a relation between two entities in the platform. " + SECURITY_CHECKS_ENTITIES_DESCRIPTION) + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/v2/relation", method = RequestMethod.DELETE, params = {FROM_ID, FROM_TYPE, RELATION_TYPE, TO_ID, TO_TYPE}) + @ResponseStatus(value = HttpStatus.OK) + public EntityRelation deleteRelationV2(@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_ID) String strFromId, + @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(FROM_TYPE) String strFromType, + @Parameter(description = RELATION_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(RELATION_TYPE) String strRelationType, + @Parameter(description = RELATION_TYPE_GROUP_PARAM_DESCRIPTION) @RequestParam(value = "relationTypeGroup", required = false) String strRelationTypeGroup, + @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @RequestParam(TO_ID) String strToId, + @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @RequestParam(TO_TYPE) String strToType) throws ThingsboardException { + return doDelete(strFromId, strFromType, strRelationType, strRelationTypeGroup, strToId, strToType); + } + + private EntityRelation doDelete(String strFromId, String strFromType, String strRelationType, String strRelationTypeGroup, String strToId, String strToType) throws ThingsboardException { checkParameter(FROM_ID, strFromId); checkParameter(FROM_TYPE, strFromType); checkParameter(RELATION_TYPE, strRelationType); diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java index f2464f3e47..84cb6f41c0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java @@ -103,7 +103,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - relation = doPost("/api/relation", relation, EntityRelation.class); + relation = doPost("/api/v2/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", mainDevice.getUuidId(), EntityType.DEVICE, @@ -315,7 +315,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Device device = buildSimpleDevice("Test device 1"); EntityRelation relation = createFromRelation(mainDevice, device, "CONTAINS"); - relation = doPost("/api/relation", relation, EntityRelation.class); + relation = doPost("/api/v2/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", mainDevice.getUuidId(), EntityType.DEVICE, @@ -329,7 +329,11 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - var deletedRelation = doDelete(url, EntityRelation.class); + String deleteUrl = String.format("/api/v2/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + mainDevice.getUuidId(), EntityType.DEVICE, + "CONTAINS", device.getUuidId(), EntityType.DEVICE + ); + var deletedRelation = doDelete(deleteUrl, EntityRelation.class); testNotifyEntityAllOneTimeRelation(deletedRelation, savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), @@ -523,7 +527,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { @Test public void testCreateRelationFromTenantToDevice() throws Exception { EntityRelation relation = new EntityRelation(tenantAdmin.getTenantId(), mainDevice.getId(), "CONTAINS"); - relation = doPost("/api/relation", relation, EntityRelation.class); + relation = doPost("/api/v2/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", tenantAdmin.getTenantId(), EntityType.TENANT, @@ -539,7 +543,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { @Test public void testCreateRelationFromDeviceToTenant() throws Exception { EntityRelation relation = new EntityRelation(mainDevice.getId(), tenantAdmin.getTenantId(), "CONTAINS"); - relation = doPost("/api/relation", relation, EntityRelation.class); + relation = doPost("/api/v2/relation", relation, EntityRelation.class); String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", mainDevice.getUuidId(), EntityType.DEVICE, diff --git a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java index baeafd3ad2..82b7a09623 100644 --- a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java @@ -48,7 +48,7 @@ public class RelationEdgeTest extends AbstractEdgeTest { relation.setTo(asset.getId()); relation.setTypeGroup(RelationTypeGroup.COMMON); edgeImitator.expectMessageAmount(1); - relation = doPost("/api/relation", relation, EntityRelation.class); + relation = doPost("/api/v2/relation", relation, EntityRelation.class); Assert.assertTrue(edgeImitator.waitForMessages()); AbstractMessage latestMessage = edgeImitator.getLatestMessage(); Assert.assertTrue(latestMessage instanceof RelationUpdateMsg); @@ -60,7 +60,7 @@ public class RelationEdgeTest extends AbstractEdgeTest { // delete relation edgeImitator.expectMessageAmount(1); - var deletedRelation = doDelete("/api/relation?" + + var deletedRelation = doDelete("/api/v2/relation?" + "fromId=" + relation.getFrom().getId().toString() + "&fromType=" + relation.getFrom().getEntityType().name() + "&relationType=" + relation.getType() + @@ -118,7 +118,7 @@ public class RelationEdgeTest extends AbstractEdgeTest { deviceToAssetRelation.setTypeGroup(RelationTypeGroup.COMMON); edgeImitator.expectMessageAmount(1); - deviceToAssetRelation = doPost("/api/relation", deviceToAssetRelation, EntityRelation.class); + deviceToAssetRelation = doPost("/api/v2/relation", deviceToAssetRelation, EntityRelation.class); Assert.assertTrue(edgeImitator.waitForMessages()); EntityRelation assetToTenantRelation = new EntityRelation(); diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index a55bcc849d..ffc88f7f5a 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -937,7 +937,7 @@ public class VersionControlTest extends AbstractControllerTest { relation.setType(EntityRelation.MANAGES_TYPE); relation.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); relation.setTypeGroup(RelationTypeGroup.COMMON); - return doPost("/api/relation", relation, EntityRelation.class); + return doPost("/api/v2/relation", relation, EntityRelation.class); } protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 8783770be3..08b2212d7c 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -1693,6 +1693,10 @@ public class RestClient implements Closeable { restTemplate.postForLocation(baseURL + "/api/relation", relation); } + public EntityRelation saveRelationV2(EntityRelation relation) { + return restTemplate.postForEntity(baseURL + "/api/v2/relation", relation, EntityRelation.class).getBody(); + } + public void deleteRelation(EntityId fromId, String relationType, RelationTypeGroup relationTypeGroup, EntityId toId) { Map params = new HashMap<>(); params.put("fromId", fromId.getId().toString()); @@ -1704,6 +1708,26 @@ public class RestClient implements Closeable { restTemplate.delete(baseURL + "/api/relation?fromId={fromId}&fromType={fromType}&relationType={relationType}&relationTypeGroup={relationTypeGroup}&toId={toId}&toType={toType}", params); } + public Optional deleteRelationV2(EntityId fromId, String relationType, RelationTypeGroup relationTypeGroup, EntityId toId) { + Map params = new HashMap<>(); + params.put("fromId", fromId.getId().toString()); + params.put("fromType", fromId.getEntityType().name()); + params.put("relationType", relationType); + params.put("relationTypeGroup", relationTypeGroup.name()); + params.put("toId", toId.getId().toString()); + params.put("toType", toId.getEntityType().name()); + try { + var relation = restTemplate.exchange(baseURL + "/api/relation?fromId={fromId}&fromType={fromType}&relationType={relationType}&relationTypeGroup={relationTypeGroup}&toId={toId}&toType={toType}", HttpMethod.DELETE, HttpEntity.EMPTY, EntityRelation.class, params); + return Optional.ofNullable(relation.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + public void deleteRelations(EntityId entityId) { restTemplate.delete(baseURL + "/api/relations?entityId={entityId}&entityType={entityType}", entityId.getId().toString(), entityId.getEntityType().name()); }