YevhenBondarenko
2 years ago
19 changed files with 837 additions and 40 deletions
@ -0,0 +1,57 @@ |
|||
#!/bin/bash |
|||
# |
|||
# Copyright © 2016-2024 The Thingsboard Authors |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|||
# you may not use this file except in compliance with the License. |
|||
# You may obtain a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, |
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
# See the License for the specific language governing permissions and |
|||
# limitations under the License. |
|||
# |
|||
|
|||
set -e # exit on any error |
|||
|
|||
#PROJECTS="msa/tb-node,msa/web-ui,rule-engine-pe/rule-node-twilio-sms" |
|||
PROJECTS="" |
|||
|
|||
if [ "$1" ]; then |
|||
PROJECTS="--projects $1" |
|||
fi |
|||
|
|||
echo "Building and pushing [amd64,arm64] projects '$PROJECTS' ..." |
|||
echo "HELP: usage ./build.sh [projects]" |
|||
echo "HELP: example ./build.sh msa/web-ui,msa/web-report" |
|||
java -version |
|||
#echo "Cleaning ui-ngx/node_modules" && rm -rf ui-ngx/node_modules |
|||
|
|||
MAVEN_OPTS="-Xmx1024m" NODE_OPTIONS="--max_old_space_size=4096" DOCKER_CLI_EXPERIMENTAL=enabled DOCKER_BUILDKIT=0 \ |
|||
mvn -T2 license:format clean install -DskipTests \ |
|||
$PROJECTS --also-make |
|||
# \ |
|||
# -Dpush-docker-amd-arm-images |
|||
# -Ddockerfile.skip=false -Dpush-docker-image=true |
|||
# --offline |
|||
# --projects '!msa/web-report' --also-make |
|||
|
|||
# push all |
|||
# mvn -T 1C license:format clean install -DskipTests -Ddockerfile.skip=false -Dpush-docker-image=true |
|||
|
|||
|
|||
## Build and push AMD and ARM docker images using docker buildx |
|||
## Reference to article how to setup docker miltiplatform build environment: https://medium.com/@artur.klauser/building-multi-architecture-docker-images-with-buildx-27d80f7e2408 |
|||
## install docker-ce from docker repo https://docs.docker.com/engine/install/ubuntu/ |
|||
# sudo apt install -y qemu-user-static binfmt-support |
|||
# export DOCKER_CLI_EXPERIMENTAL=enabled |
|||
# docker version |
|||
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes |
|||
# docker buildx create --name mybuilder |
|||
# docker buildx use mybuilder |
|||
# docker buildx inspect --bootstrap |
|||
# docker buildx ls |
|||
# mvn clean install -P push-docker-amd-arm-images |
|||
@ -0,0 +1,35 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.cache; |
|||
|
|||
import org.springframework.data.redis.serializer.RedisSerializer; |
|||
import org.springframework.data.redis.serializer.SerializationException; |
|||
|
|||
public class TbJavaRedisSerializer<K, V> implements TbRedisSerializer<K, V> { |
|||
|
|||
final RedisSerializer<Object> serializer = RedisSerializer.java(); |
|||
|
|||
@Override |
|||
public byte[] serialize(V value) throws SerializationException { |
|||
return serializer.serialize(value); |
|||
} |
|||
|
|||
@Override |
|||
public V deserialize(K key, byte[] bytes) throws SerializationException { |
|||
return (V) serializer.deserialize(bytes); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,26 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
|
|||
import java.lang.annotation.Retention; |
|||
import java.lang.annotation.RetentionPolicy; |
|||
|
|||
@Retention(RetentionPolicy.RUNTIME) |
|||
@ConditionalOnExpression("('${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale') && '${cache.ts_latest.enabled:false}'=='true' && '${cache.type:caffeine}'=='redis' ") |
|||
public @interface SqlTsLatestAnyDaoCachedRedis { |
|||
} |
|||
@ -0,0 +1,181 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.sqlts; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import jakarta.annotation.PostConstruct; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.context.annotation.Primary; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.server.cache.TbCacheValueWrapper; |
|||
import org.thingsboard.server.cache.TbTransactionalCache; |
|||
import org.thingsboard.server.common.data.id.DeviceProfileId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; |
|||
import org.thingsboard.server.common.stats.DefaultCounter; |
|||
import org.thingsboard.server.common.stats.StatsFactory; |
|||
import org.thingsboard.server.dao.cache.CacheExecutorService; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; |
|||
import org.thingsboard.server.dao.timeseries.TsLatestCacheKey; |
|||
import org.thingsboard.server.dao.util.SqlTsLatestAnyDaoCachedRedis; |
|||
|
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
|
|||
@Slf4j |
|||
@Component |
|||
@SqlTsLatestAnyDaoCachedRedis |
|||
@RequiredArgsConstructor |
|||
@Primary |
|||
public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao implements TimeseriesLatestDao { |
|||
public static final String STATS_NAME = "ts_latest.cache"; |
|||
final CacheExecutorService cacheExecutorService; |
|||
final SqlTimeseriesLatestDao sqlDao; |
|||
final StatsFactory statsFactory; |
|||
final TbTransactionalCache<TsLatestCacheKey, TsKvEntry> cache; |
|||
DefaultCounter hitCounter; |
|||
DefaultCounter missCounter; |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
log.info("Init Redis cache-aside SQL Timeseries Latest DAO"); |
|||
this.hitCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "hit"); |
|||
this.missCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "miss"); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Void> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { |
|||
ListenableFuture<Void> future = sqlDao.saveLatest(tenantId, entityId, tsKvEntry); |
|||
future = Futures.transform(future, x -> { |
|||
cache.put(new TsLatestCacheKey(entityId, tsKvEntry.getKey()), tsKvEntry); |
|||
return x; |
|||
}, |
|||
cacheExecutorService); |
|||
if (log.isTraceEnabled()) { |
|||
Futures.addCallback(future, new FutureCallback<>() { |
|||
@Override |
|||
public void onSuccess(Void result) { |
|||
log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.info("saveLatest onFailure [{}][{}][{}]", entityId, tsKvEntry.getKey(), tsKvEntry, t); |
|||
} |
|||
}, MoreExecutors.directExecutor()); |
|||
} |
|||
return future; |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<TsKvLatestRemovingResult> removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { |
|||
ListenableFuture<TsKvLatestRemovingResult> future = sqlDao.removeLatest(tenantId, entityId, query); |
|||
future = Futures.transform(future, x -> { |
|||
cache.evict(new TsLatestCacheKey(entityId, query.getKey())); |
|||
return x; |
|||
}, |
|||
cacheExecutorService); |
|||
if (log.isTraceEnabled()) { |
|||
Futures.addCallback(future, new FutureCallback<>() { |
|||
@Override |
|||
public void onSuccess(TsKvLatestRemovingResult result) { |
|||
log.trace("removeLatest onSuccess [{}][{}][{}]", entityId, query.getKey(), query); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.info("removeLatest onFailure [{}][{}][{}]", entityId, query.getKey(), query, t); |
|||
} |
|||
}, MoreExecutors.directExecutor()); |
|||
} |
|||
return future; |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Optional<TsKvEntry>> findLatestOpt(TenantId tenantId, EntityId entityId, String key) { |
|||
log.trace("findLatestOpt"); |
|||
return doFindLatest(tenantId, entityId, key); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<TsKvEntry> findLatest(TenantId tenantId, EntityId entityId, String key) { |
|||
return Futures.transform(doFindLatest(tenantId, entityId, key), x -> sqlDao.wrapNullTsKvEntry(key, x.orElse(null)), MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
public ListenableFuture<Optional<TsKvEntry>> doFindLatest(TenantId tenantId, EntityId entityId, String key) { |
|||
final TsLatestCacheKey cacheKey = new TsLatestCacheKey(entityId, key); |
|||
ListenableFuture<TbCacheValueWrapper<TsKvEntry>> cacheFuture = cacheExecutorService.submit(() -> cache.get(cacheKey)); |
|||
|
|||
return Futures.transformAsync(cacheFuture, (cacheValueWrap) -> { |
|||
if (cacheValueWrap != null) { |
|||
final TsKvEntry tsKvEntry = cacheValueWrap.get(); |
|||
log.debug("findLatest cache hit [{}][{}][{}]", entityId, key, tsKvEntry); |
|||
return Futures.immediateFuture(Optional.ofNullable(tsKvEntry)); |
|||
} |
|||
log.debug("findLatest cache miss [{}][{}]", entityId, key); |
|||
ListenableFuture<Optional<TsKvEntry>> daoFuture = sqlDao.findLatestOpt(tenantId,entityId, key); |
|||
|
|||
return Futures.transformAsync(daoFuture, (daoValue) -> { |
|||
|
|||
if (daoValue.isEmpty()) { |
|||
//TODO implement the cache logic if no latest found in TS DAO. Currently we are always getting from DB to stay on the safe side
|
|||
return Futures.immediateFuture(daoValue); |
|||
} |
|||
ListenableFuture<Optional<TsKvEntry>> cachePutFuture = cacheExecutorService.submit(() -> { |
|||
cache.put(new TsLatestCacheKey(entityId, key), daoValue.get()); |
|||
return daoValue; |
|||
}); |
|||
|
|||
Futures.addCallback(cachePutFuture, new FutureCallback<>() { |
|||
@Override |
|||
public void onSuccess(Optional<TsKvEntry> result) { |
|||
log.trace("saveLatest onSuccess [{}][{}][{}]", entityId, key, result); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.info("saveLatest onFailure [{}][{}][{}]", entityId, key, daoValue, t); |
|||
} |
|||
|
|||
}, MoreExecutors.directExecutor()); |
|||
return cachePutFuture; |
|||
}, MoreExecutors.directExecutor()); |
|||
}, MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) { |
|||
return sqlDao.findAllLatest(tenantId, entityId); |
|||
} |
|||
|
|||
@Override |
|||
public List<String> findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { |
|||
return sqlDao.findAllKeysByDeviceProfileId(tenantId, deviceProfileId); |
|||
} |
|||
|
|||
@Override |
|||
public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds) { |
|||
return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.timeseries; |
|||
|
|||
import lombok.AllArgsConstructor; |
|||
import lombok.EqualsAndHashCode; |
|||
import lombok.Getter; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
|
|||
import java.io.Serial; |
|||
import java.io.Serializable; |
|||
|
|||
@EqualsAndHashCode |
|||
@Getter |
|||
@AllArgsConstructor |
|||
public class TsLatestCacheKey implements Serializable { |
|||
private static final long serialVersionUID = 2024369077925351881L; |
|||
|
|||
private final EntityId entityId; |
|||
private final String key; |
|||
|
|||
@Override |
|||
public String toString() { |
|||
return "{" + entityId + "}" + key; |
|||
} |
|||
} |
|||
@ -0,0 +1,148 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.timeseries; |
|||
|
|||
import jakarta.annotation.PostConstruct; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.NotImplementedException; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
import org.springframework.dao.InvalidDataAccessApiUsageException; |
|||
import org.springframework.data.redis.connection.RedisConnection; |
|||
import org.springframework.data.redis.connection.RedisConnectionFactory; |
|||
import org.springframework.data.redis.connection.ReturnType; |
|||
import org.springframework.data.redis.serializer.StringRedisSerializer; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.cache.CacheSpecsMap; |
|||
import org.thingsboard.server.cache.RedisTbTransactionalCache; |
|||
import org.thingsboard.server.cache.TBRedisCacheConfiguration; |
|||
import org.thingsboard.server.cache.TbCacheTransaction; |
|||
import org.thingsboard.server.cache.TbCacheValueWrapper; |
|||
import org.thingsboard.server.cache.TbJavaRedisSerializer; |
|||
import org.thingsboard.server.common.data.CacheConstants; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
|
|||
import java.io.Serializable; |
|||
import java.util.Arrays; |
|||
import java.util.Collection; |
|||
import java.util.List; |
|||
import java.util.Set; |
|||
|
|||
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") |
|||
@Service("TsLatestCache") |
|||
@Slf4j |
|||
public class TsLatestRedisCache<K extends Serializable, V extends Serializable> extends RedisTbTransactionalCache<TsLatestCacheKey, TsKvEntry> { |
|||
|
|||
static final byte[] UPSERT_TS_LATEST_LUA_SCRIPT = StringRedisSerializer.UTF_8.serialize("" + |
|||
"redis.call('ZREMRANGEBYSCORE', KEYS[1], ARGV[1], ARGV[1]); " + |
|||
"redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2]); " + |
|||
"local current_size = redis.call('ZCARD', KEYS[1]); " + |
|||
"if current_size > 1 then" + |
|||
" redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -2) " + |
|||
"end;"); |
|||
static final byte[] UPSERT_TS_LATEST_SHA = StringRedisSerializer.UTF_8.serialize("24e226c3ea34e3e850113e8eb1f3cd2b88171988"); |
|||
|
|||
public TsLatestRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { |
|||
super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJavaRedisSerializer<>()); |
|||
} |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
try (var connection = getConnection(UPSERT_TS_LATEST_SHA)) { |
|||
log.debug("Loading LUA with expected SHA[{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), connection.getNativeConnection()); |
|||
String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT); |
|||
if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { |
|||
log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), sha, connection.getNativeConnection()); |
|||
} |
|||
} catch (Throwable t) { |
|||
log.error("Error on Redis TS Latest cache init", t); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public TbCacheValueWrapper<TsKvEntry> get(TsLatestCacheKey key) { |
|||
log.debug("get [{}]", key); |
|||
return super.get(key); |
|||
} |
|||
|
|||
@Override |
|||
protected byte[] doGet(RedisConnection connection, byte[] rawKey) { |
|||
log.trace("doGet [{}][{}]", connection, rawKey); |
|||
Set<byte[]> values = connection.commands().zRange(rawKey, -1, -1); |
|||
return values == null ? null : values.stream().findFirst().orElse(null); |
|||
} |
|||
|
|||
@Override |
|||
public void put(TsLatestCacheKey key, TsKvEntry value) { |
|||
log.trace("put [{}][{}]", key, value); |
|||
final byte[] rawKey = getRawKey(key); |
|||
try (var connection = getConnection(rawKey)) { |
|||
byte[] rawValue = getRawValue(value); |
|||
byte[] ts = StringRedisSerializer.UTF_8.serialize(String.valueOf(value.toTsValue().getTs())); |
|||
try { |
|||
connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue); |
|||
} catch (InvalidDataAccessApiUsageException e) { |
|||
log.debug("loading LUA [{}]", connection.getNativeConnection()); |
|||
String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT); |
|||
if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) { |
|||
log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}]", new String(UPSERT_TS_LATEST_SHA), sha); |
|||
} |
|||
try { |
|||
connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue); |
|||
} catch (InvalidDataAccessApiUsageException ignored) { |
|||
log.debug("Slowly executing eval instead of fast evalsha"); |
|||
connection.scriptingCommands().eval(UPSERT_TS_LATEST_LUA_SCRIPT, ReturnType.VALUE, 1, rawKey, ts, rawValue); |
|||
} |
|||
|
|||
} |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void evict(TsLatestCacheKey key) { |
|||
log.trace("evict [{}]", key); |
|||
final byte[] rawKey = getRawKey(key); |
|||
try (var connection = getConnection(rawKey)) { |
|||
connection.keyCommands().del(rawKey); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void putIfAbsent(TsLatestCacheKey key, TsKvEntry value) { |
|||
log.trace("putIfAbsent [{}][{}]", key, value); |
|||
throw new NotImplementedException("putIfAbsent is not supported by TsLatestRedisCache"); |
|||
} |
|||
|
|||
@Override |
|||
public void evict(Collection<TsLatestCacheKey> keys) { |
|||
throw new NotImplementedException("evict by many keys is not supported by TsLatestRedisCache"); |
|||
} |
|||
|
|||
@Override |
|||
public void evictOrPut(TsLatestCacheKey key, TsKvEntry value) { |
|||
throw new NotImplementedException("evictOrPut is not supported by TsLatestRedisCache"); |
|||
} |
|||
|
|||
@Override |
|||
public TbCacheTransaction<TsLatestCacheKey, TsKvEntry> newTransactionForKey(TsLatestCacheKey key) { |
|||
throw new NotImplementedException("newTransactionForKey is not supported by TsLatestRedisCache"); |
|||
} |
|||
|
|||
@Override |
|||
public TbCacheTransaction<TsLatestCacheKey, TsKvEntry> newTransactionForKeys(List<TsLatestCacheKey> keys) { |
|||
throw new NotImplementedException("newTransactionForKeys is not supported by TsLatestRedisCache"); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,90 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.junit.ClassRule; |
|||
import org.junit.rules.ExternalResource; |
|||
import org.testcontainers.containers.GenericContainer; |
|||
import org.testcontainers.containers.Network; |
|||
import org.testcontainers.containers.output.OutputFrame; |
|||
import redis.clients.jedis.Jedis; |
|||
|
|||
import java.util.List; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
@Slf4j |
|||
public class AbstractRedisClusterContainer { |
|||
|
|||
static final String nodes = "127.0.0.1:6371,127.0.0.1:6372,127.0.0.1:6373,127.0.0.1:6374,127.0.0.1:6375,127.0.0.1:6376"; |
|||
|
|||
@ClassRule(order = 0) |
|||
public static Network network = Network.newNetwork(); |
|||
@ClassRule(order = 1) |
|||
public static GenericContainer redis1 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6371").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); |
|||
@ClassRule(order = 2) |
|||
public static GenericContainer redis2 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6372").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); |
|||
@ClassRule(order = 3) |
|||
public static GenericContainer redis3 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6373").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); |
|||
@ClassRule(order = 4) |
|||
public static GenericContainer redis4 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6374").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); |
|||
@ClassRule(order = 5) |
|||
public static GenericContainer redis5 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6375").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); |
|||
@ClassRule(order = 6) |
|||
public static GenericContainer redis6 = new GenericContainer("bitnami/redis-cluster:latest").withEnv("REDIS_PORT_NUMBER","6376").withNetworkMode("host").withLogConsumer(x->log.warn("{}", ((OutputFrame)x).getUtf8StringWithoutLineEnding())).withEnv("ALLOW_EMPTY_PASSWORD","yes").withEnv("REDIS_NODES",nodes); |
|||
|
|||
|
|||
@ClassRule(order = 100) |
|||
public static ExternalResource resource = new ExternalResource() { |
|||
@Override |
|||
protected void before() throws Throwable { |
|||
redis1.start(); |
|||
redis2.start(); |
|||
redis3.start(); |
|||
redis4.start(); |
|||
redis5.start(); |
|||
redis6.start(); |
|||
|
|||
Thread.sleep(TimeUnit.SECONDS.toMillis(5)); // otherwise not all containers have time to start
|
|||
|
|||
String clusterCreateCommand = "echo yes | redis-cli --cluster create " + |
|||
"127.0.0.1:6371 127.0.0.1:6372 127.0.0.1:6373 127.0.0.1:6374 127.0.0.1:6375 127.0.0.1:6376 " + |
|||
"--cluster-replicas 1"; |
|||
log.warn("Command to init Redis Cluster: {}", clusterCreateCommand); |
|||
var result = redis6.execInContainer("/bin/sh", "-c", clusterCreateCommand); |
|||
log.warn("Init cluster result: {}", result); |
|||
|
|||
log.warn("Connect to nodes: {}", nodes); |
|||
System.setProperty("cache.type", "redis"); |
|||
System.setProperty("redis.connection.type", "cluster"); |
|||
System.setProperty("redis.cluster.nodes", nodes); |
|||
System.setProperty("redis.cluster.useDefaultPoolConfig", "false"); |
|||
} |
|||
|
|||
@Override |
|||
protected void after() { |
|||
redis1.stop(); |
|||
redis2.stop(); |
|||
redis3.stop(); |
|||
redis4.stop(); |
|||
redis5.stop(); |
|||
redis6.stop(); |
|||
List.of("cache.type", "redis.connection.type", "redis.cluster.nodes", "redis.cluster.useDefaultPoolConfig\"") |
|||
.forEach(System.getProperties()::remove); |
|||
} |
|||
}; |
|||
|
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao; |
|||
|
|||
import org.junit.extensions.cpsuite.ClasspathSuite; |
|||
import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; |
|||
import org.junit.runner.RunWith; |
|||
|
|||
@RunWith(ClasspathSuite.class) |
|||
@ClassnameFilters( |
|||
//All the same tests using redis instead of caffeine.
|
|||
{ |
|||
"org.thingsboard.server.dao.service.*ServiceSqlTest", |
|||
} |
|||
) |
|||
public class RedisClusterSqlTestSuite extends AbstractRedisClusterContainer { |
|||
|
|||
} |
|||
@ -0,0 +1,64 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.junit.jupiter.api.AfterAll; |
|||
import org.junit.jupiter.api.BeforeAll; |
|||
import org.junit.jupiter.api.Test; |
|||
import org.testcontainers.containers.GenericContainer; |
|||
import org.testcontainers.containers.output.OutputFrame; |
|||
import org.testcontainers.junit.jupiter.Container; |
|||
import org.testcontainers.junit.jupiter.Testcontainers; |
|||
|
|||
import java.util.List; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
|
|||
@Testcontainers |
|||
@Slf4j |
|||
public class RedisJUnit5Test { |
|||
|
|||
@Container |
|||
private static final GenericContainer REDIS = new GenericContainer("redis:7.2-bookworm") |
|||
.withLogConsumer(s -> log.error(((OutputFrame) s).getUtf8String().trim())) |
|||
.withExposedPorts(6379); |
|||
|
|||
@BeforeAll |
|||
static void beforeAll() { |
|||
log.warn("Starting redis..."); |
|||
REDIS.start(); |
|||
System.setProperty("cache.type", "redis"); |
|||
System.setProperty("redis.connection.type", "standalone"); |
|||
System.setProperty("redis.standalone.host", REDIS.getHost()); |
|||
System.setProperty("redis.standalone.port", String.valueOf(REDIS.getMappedPort(6379))); |
|||
|
|||
} |
|||
|
|||
@AfterAll |
|||
static void afterAll() { |
|||
List.of("cache.type", "redis.connection.type", "redis.standalone.host", "redis.standalone.port") |
|||
.forEach(System.getProperties()::remove); |
|||
REDIS.stop(); |
|||
log.warn("Redis is stopped"); |
|||
} |
|||
|
|||
@Test |
|||
void test() { |
|||
assertThat(REDIS.isRunning()).isTrue(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,45 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.timeseries; |
|||
|
|||
import lombok.SneakyThrows; |
|||
import org.junit.jupiter.api.Test; |
|||
|
|||
import java.security.MessageDigest; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
|
|||
class TsLatestRedisCacheTest { |
|||
|
|||
@Test |
|||
void testUpsertTsLatestLUAScriptHash() { |
|||
assertThat(getSHA1(TsLatestRedisCache.UPSERT_TS_LATEST_LUA_SCRIPT)).isEqualTo(new String(TsLatestRedisCache.UPSERT_TS_LATEST_SHA)); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
String getSHA1(byte[] script) { |
|||
MessageDigest md = MessageDigest.getInstance("SHA-1"); |
|||
byte[] hash = md.digest(script); |
|||
|
|||
StringBuilder sb = new StringBuilder(); |
|||
for (byte b : hash) { |
|||
sb.append(String.format("%02x", b)); |
|||
} |
|||
|
|||
return sb.toString(); |
|||
} |
|||
|
|||
} |
|||
Loading…
Reference in new issue