tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

TsLatestRedisCache: added LUA upsert script. load script, eval, evalsha, test for script sha, fetch latest by zRange Redis command

pull/10584/head
Sergey Matvienko 2 years ago
parent
c49d97f7d1
commit
a4469f6953
3 changed files with 91 additions and 15 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java
  2. 75
      dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java
  3. 30
      dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java

1
common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java
View File

@ -51,6 +51,7 @@ public abstract class RedisTbTransactionalCache<K extends Serializable, V extend
@Getter
private final String cacheName;
@Getter
private final JedisConnectionFactory connectionFactory;
private final RedisSerializer<String> keySerializer = StringRedisSerializer.UTF_8;
private final TbRedisSerializer<K, V> valueSerializer;

75
dao/src/main/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCache.java
View File

@ -15,12 +15,15 @@
*/
package org.thingsboard.server.dao.timeseries;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.NotImplementedException;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.dao.InvalidDataAccessApiUsageException;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.RedisStringCommands;
import org.springframework.data.redis.connection.ReturnType;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CacheSpecsMap;
import org.thingsboard.server.cache.RedisTbTransactionalCache;
@ -32,33 +35,42 @@ import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Set;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@Service("TsLatestCache")
@Slf4j
public class TsLatestRedisCache<K extends Serializable, V extends Serializable> extends RedisTbTransactionalCache<TsLatestCacheKey, TsKvEntry> {
static final byte[] UPSERT_TS_LATEST_LUA_SCRIPT = StringRedisSerializer.UTF_8.serialize("" +
"redis.call('ZREMRANGEBYSCORE', KEYS[1], ARGV[1], ARGV[1]); " +
"redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2]); " +
"local current_size = redis.call('ZCARD', KEYS[1]); " +
"if current_size > 1 then" +
" redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -2) " +
"end;");
static final byte[] UPSERT_TS_LATEST_SHA = StringRedisSerializer.UTF_8.serialize("24e226c3ea34e3e850113e8eb1f3cd2b88171988");
public TsLatestRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.TS_LATEST_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbJavaRedisSerializer<>());
}
@Override
public void put(TsLatestCacheKey key, TsKvEntry value) {
log.trace("put [{}][{}]", key, value);
final byte[] rawKey = getRawKey(key);
try (var connection = getConnection(rawKey)) {
put(connection, rawKey, value, RedisStringCommands.SetOption.UPSERT);
@PostConstruct
public void init() {
try (var connection = getConnection(UPSERT_TS_LATEST_SHA)) {
log.debug("Loading LUA with expected SHA[{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), connection.getNativeConnection());
String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT);
if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) {
log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}], connection [{}]", new String(UPSERT_TS_LATEST_SHA), sha, connection.getNativeConnection());
}
} catch (Throwable t) {
log.error("Error on Redis TS Latest cache init", t);
}
}
@Override
public void putIfAbsent(TsLatestCacheKey key, TsKvEntry value) {
log.trace("putIfAbsent [{}][{}]", key, value);
throw new NotImplementedException("putIfAbsent is not supported by TsLatestRedisCache");
}
@Override
public TbCacheValueWrapper<TsKvEntry> get(TsLatestCacheKey key) {
log.debug("get [{}]", key);
@ -68,11 +80,38 @@ public class TsLatestRedisCache<K extends Serializable, V extends Serializable>
@Override
protected byte[] doGet(RedisConnection connection, byte[] rawKey) {
log.trace("doGet [{}][{}]", connection, rawKey);
return connection.stringCommands().get(rawKey);
Set<byte[]> values = connection.commands().zRange(rawKey, -1, -1);
return values == null ? null : values.stream().findFirst().orElse(null);
}
@Override
public void evict(TsLatestCacheKey key){
public void put(TsLatestCacheKey key, TsKvEntry value) {
log.trace("put [{}][{}]", key, value);
final byte[] rawKey = getRawKey(key);
try (var connection = getConnection(rawKey)) {
byte[] rawValue = getRawValue(value);
byte[] ts = StringRedisSerializer.UTF_8.serialize(String.valueOf(value.toTsValue().getTs()));
try {
connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue);
} catch (InvalidDataAccessApiUsageException e) {
log.debug("loading LUA [{}]", connection.getNativeConnection());
String sha = connection.scriptingCommands().scriptLoad(UPSERT_TS_LATEST_LUA_SCRIPT);
if (!Arrays.equals(UPSERT_TS_LATEST_SHA, StringRedisSerializer.UTF_8.serialize(sha))) {
log.error("SHA for UPSERT_TS_LATEST_LUA_SCRIPT wrong! Expected [{}], but actual [{}]", new String(UPSERT_TS_LATEST_SHA), sha);
}
try {
connection.scriptingCommands().evalSha(UPSERT_TS_LATEST_SHA, ReturnType.VALUE, 1, rawKey, ts, rawValue);
} catch (InvalidDataAccessApiUsageException ignored) {
log.debug("Slowly executing eval instead of fast evalsha");
connection.scriptingCommands().eval(UPSERT_TS_LATEST_LUA_SCRIPT, ReturnType.VALUE, 1, rawKey, ts, rawValue);
}
}
}
}
@Override
public void evict(TsLatestCacheKey key) {
log.trace("evict [{}]", key);
final byte[] rawKey = getRawKey(key);
try (var connection = getConnection(rawKey)) {
@ -80,6 +119,12 @@ public class TsLatestRedisCache<K extends Serializable, V extends Serializable>
}
}
@Override
public void putIfAbsent(TsLatestCacheKey key, TsKvEntry value) {
log.trace("putIfAbsent [{}][{}]", key, value);
throw new NotImplementedException("putIfAbsent is not supported by TsLatestRedisCache");
}
@Override
public void evict(Collection<TsLatestCacheKey> keys) {
throw new NotImplementedException("evict by many keys is not supported by TsLatestRedisCache");

30
dao/src/test/java/org/thingsboard/server/dao/timeseries/TsLatestRedisCacheTest.java
View File

@ -0,0 +1,30 @@
package org.thingsboard.server.dao.timeseries;
import lombok.SneakyThrows;
import org.junit.jupiter.api.Test;
import java.security.MessageDigest;
import static org.assertj.core.api.Assertions.assertThat;
class TsLatestRedisCacheTest {
@Test
void testUpsertTsLatestLUAScriptHash() {
assertThat(getSHA1(TsLatestRedisCache.UPSERT_TS_LATEST_LUA_SCRIPT)).isEqualTo(new String(TsLatestRedisCache.UPSERT_TS_LATEST_SHA));
}
@SneakyThrows
String getSHA1(byte[] script) {
MessageDigest md = MessageDigest.getInstance("SHA-1");
byte[] hash = md.digest(script);
StringBuilder sb = new StringBuilder();
for (byte b : hash) {
sb.append(String.format("%02x", b));
}
return sb.toString();
}
}
Write Preview
Loading…
Cancel
Save