diff --git a/application/pom.xml b/application/pom.xml index 98a65bc5bb..4330300adf 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -369,14 +369,11 @@ org.apache.maven.plugins maven-surefire-plugin - ${surfire.version} thingsboard - **/sql/*Test.java - **/psql/*Test.java **/nosql/*Test.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index 0b28aca7b0..b0830317a7 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -884,10 +884,10 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { return; } log.debug("[{}] Restoring sessions from cache", deviceId); - DeviceSessionsCacheEntry sessionsDump = null; + DeviceSessionsCacheEntry sessionsDump; try { - sessionsDump = DeviceSessionsCacheEntry.parseFrom(systemContext.getDeviceSessionCacheService().get(deviceId)); - } catch (InvalidProtocolBufferException e) { + sessionsDump = systemContext.getDeviceSessionCacheService().get(deviceId); + } catch (Exception e) { log.warn("[{}] Failed to decode device sessions from cache", deviceId); return; } @@ -942,7 +942,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { }); systemContext.getDeviceSessionCacheService() .put(deviceId, DeviceSessionsCacheEntry.newBuilder() - .addAllSessions(sessionsList).build().toByteArray()); + .addAllSessions(sessionsList).build()); } void init(TbActorCtx ctx) { diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java index ad734a5c07..7fd42740c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceProvisionServiceImpl.java @@ -210,7 +210,7 @@ public class DeviceProvisionServiceImpl implements DeviceProvisionService { } } - private ListenableFuture> saveProvisionStateAttribute(Device device) { + private ListenableFuture> saveProvisionStateAttribute(Device device) { return attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry(DEVICE_PROVISION_STATE, PROVISIONED_STATE), System.currentTimeMillis()))); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index de0cea14d6..6c991444b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -309,10 +309,10 @@ public final class EdgeGrpcSession implements Closeable { public void onSuccess(@Nullable UUID ifOffset) { if (ifOffset != null) { Long newStartTs = Uuids.unixTimestamp(ifOffset); - ListenableFuture> updateFuture = updateQueueStartTs(newStartTs); + ListenableFuture> updateFuture = updateQueueStartTs(newStartTs); Futures.addCallback(updateFuture, new FutureCallback<>() { @Override - public void onSuccess(@Nullable List list) { + public void onSuccess(@Nullable List list) { log.debug("[{}] queue offset was updated [{}][{}]", sessionId, ifOffset, newStartTs); result.set(null); } @@ -496,7 +496,7 @@ public final class EdgeGrpcSession implements Closeable { }, ctx.getGrpcCallbackExecutorService()); } - private ListenableFuture> updateQueueStartTs(Long newStartTs) { + private ListenableFuture> updateQueueStartTs(Long newStartTs) { log.trace("[{}] updating QueueStartTs [{}][{}]", this.sessionId, edge.getId(), newStartTs); List attributes = Collections.singletonList( new BaseAttributeKvEntry( diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java index 9a9b568a23..a72f41c887 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java @@ -206,10 +206,10 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { SettableFuture futureToSet = SettableFuture.create(); JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); Set attributes = JsonConverter.convertToAttributes(json); - ListenableFuture> future = attributesService.save(tenantId, entityId, metaData.getValue("scope"), new ArrayList<>(attributes)); - Futures.addCallback(future, new FutureCallback>() { + ListenableFuture> future = attributesService.save(tenantId, entityId, metaData.getValue("scope"), new ArrayList<>(attributes)); + Futures.addCallback(future, new FutureCallback<>() { @Override - public void onSuccess(@Nullable List voids) { + public void onSuccess(@Nullable List keys) { Pair defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId); String queueName = defaultQueueAndRuleChain.getKey(); RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index c5bdf722af..9e7636879f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -532,7 +532,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } else { - ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, + ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) , System.currentTimeMillis()))); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index c41eebdbb6..7a5bc08236 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -68,6 +68,10 @@ public class DefaultCacheCleanupService implements CacheCleanupService { log.info("Clear cache to upgrade from version 3.3.3 to 3.3.4 ..."); clearAll(); break; + case "3.3.4": + log.info("Clear cache to upgrade from version 3.3.4 to 3.4.0 ..."); + clearAll(); + break; default: //Do nothing, since cache cleanup is optional. } diff --git a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java index bf8af38e5f..9e5667aa19 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DefaultDeviceSessionCacheService.java @@ -15,14 +15,18 @@ */ package org.thingsboard.server.service.session; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CachePut; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheEntry; import org.thingsboard.server.queue.util.TbCoreComponent; +import java.io.Serializable; import java.util.Collections; import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; @@ -35,17 +39,20 @@ import static org.thingsboard.server.common.data.CacheConstants.SESSIONS_CACHE; @Slf4j public class DefaultDeviceSessionCacheService implements DeviceSessionCacheService { + @Autowired + protected TbTransactionalCache cache; + @Override - @Cacheable(cacheNames = SESSIONS_CACHE, key = "#deviceId.toString()") - public byte[] get(DeviceId deviceId) { + public DeviceSessionsCacheEntry get(DeviceId deviceId) { log.debug("[{}] Fetching session data from cache", deviceId); - return DeviceSessionsCacheEntry.newBuilder().addAllSessions(Collections.emptyList()).build().toByteArray(); + return cache.getAndPutInTransaction(deviceId, () -> + DeviceSessionsCacheEntry.newBuilder().addAllSessions(Collections.emptyList()).build(), false); } @Override - @CachePut(cacheNames = SESSIONS_CACHE, key = "#deviceId.toString()") - public byte[] put(DeviceId deviceId, byte[] sessions) { + public DeviceSessionsCacheEntry put(DeviceId deviceId, DeviceSessionsCacheEntry sessions) { log.debug("[{}] Pushing session data to cache: {}", deviceId, sessions); + cache.putIfAbsent(deviceId, sessions); return sessions; } } diff --git a/application/src/main/java/org/thingsboard/server/service/session/DeviceSessionCacheService.java b/application/src/main/java/org/thingsboard/server/service/session/DeviceSessionCacheService.java index 158666048c..b01f94306a 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/DeviceSessionCacheService.java +++ b/application/src/main/java/org/thingsboard/server/service/session/DeviceSessionCacheService.java @@ -23,8 +23,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.DeviceSessionsCacheE */ public interface DeviceSessionCacheService { - byte[] get(DeviceId deviceId); + DeviceSessionsCacheEntry get(DeviceId deviceId); - byte[] put(DeviceId deviceId, byte[] sessions); + DeviceSessionsCacheEntry put(DeviceId deviceId, DeviceSessionsCacheEntry sessions); } diff --git a/application/src/main/java/org/thingsboard/server/service/session/SessionCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/session/SessionCaffeineCache.java new file mode 100644 index 0000000000..92eba5c4e2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/session/SessionCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.session; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("SessionCache") +public class SessionCaffeineCache extends CaffeineTbTransactionalCache { + + public SessionCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.SESSIONS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java new file mode 100644 index 0000000000..cd3e8cb56b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.session; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("SessionCache") +public class SessionRedisCache extends RedisTbTransactionalCache { + + public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { + @Override + public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException { + return deviceSessionsCacheEntry.toByteArray(); + } + + @Override + public TransportProtos.DeviceSessionsCacheEntry deserialize(byte[] bytes) throws SerializationException { + try { + return TransportProtos.DeviceSessionsCacheEntry.parseFrom(bytes); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException("Failed to deserialize session cache entry"); + } + } + }); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 440c9075e8..38275c09b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -243,7 +243,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, FutureCallback callback) { - ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); + ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); } @@ -269,7 +269,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAndNotifyInternal(TenantId tenantId, EntityId entityId, String scope, List keys, FutureCallback callback) { - ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); + ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys); addVoidCallback(deleteFuture, callback); addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys)); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0aa7b3bb66..03dbb49763 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -387,8 +387,6 @@ cache: attributes: # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" - -caffeine: specs: relations: timeToLiveInMinutes: "${CACHE_SPECS_RELATIONS_TTL:1440}" @@ -475,6 +473,8 @@ redis: maxWaitMills: "${REDIS_POOL_CONFIG_MAX_WAIT_MS:60000}" numberTestsPerEvictionRun: "${REDIS_POOL_CONFIG_NUMBER_TESTS_PER_EVICTION_RUN:3}" blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" + # TTL for short-living SET commands that are used to replace DEL in order to enable transaction support + evictTtlInMs: "${REDIS_EVICT_TTL_MS:60000}" # Check new version updates parameters updates: diff --git a/application/src/test/java/org/thingsboard/server/cache/CaffeineCacheDefaultConfigurationTest.java b/application/src/test/java/org/thingsboard/server/cache/CaffeineCacheDefaultConfigurationTest.java index 77d36878ac..9893f08aff 100644 --- a/application/src/test/java/org/thingsboard/server/cache/CaffeineCacheDefaultConfigurationTest.java +++ b/application/src/test/java/org/thingsboard/server/cache/CaffeineCacheDefaultConfigurationTest.java @@ -36,15 +36,15 @@ import static org.assertj.core.api.Assertions.assertThat; public class CaffeineCacheDefaultConfigurationTest { @Autowired - CaffeineCacheConfiguration caffeineCacheConfiguration; + CacheSpecsMap cacheSpecsMap; @Test public void verifyTransactionAwareCacheManagerProxy() { - assertThat(caffeineCacheConfiguration.getSpecs()).as("specs").isNotNull(); - caffeineCacheConfiguration.getSpecs().forEach((name, cacheSpecs)->assertThat(cacheSpecs).as("cache %s specs", name).isNotNull()); + assertThat(cacheSpecsMap.getSpecs()).as("specs").isNotNull(); + cacheSpecsMap.getSpecs().forEach((name, cacheSpecs)->assertThat(cacheSpecs).as("cache %s specs", name).isNotNull()); SoftAssertions softly = new SoftAssertions(); - caffeineCacheConfiguration.getSpecs().forEach((name, cacheSpecs)->{ + cacheSpecsMap.getSpecs().forEach((name, cacheSpecs)->{ softly.assertThat(name).as("cache name").isNotEmpty(); softly.assertThat(cacheSpecs.getTimeToLiveInMinutes()).as("cache %s time to live", name).isGreaterThan(0); softly.assertThat(cacheSpecs.getMaxSize()).as("cache %s max size", name).isGreaterThan(0); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java index 79e19d5c82..ab902b82c6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.id.EntityId; @@ -35,6 +36,9 @@ import java.util.function.Predicate; /** * Created by ashvayka on 20.03.18. */ +@TestPropertySource(properties = { + "js.evaluator=mock", +}) public abstract class AbstractRuleEngineControllerTest extends AbstractControllerTest { @Autowired @@ -57,12 +61,18 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle } protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { + return getEvents(tenantId, entityId, DataConstants.DEBUG_RULE_NODE, limit); + } + + protected PageData getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception { TimePageLink pageLink = new TimePageLink(limit); return doGetTypedWithTimePageLink("/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&", new TypeReference>() { - }, pageLink, entityId.getEntityType(), entityId.getId(), DataConstants.DEBUG_RULE_NODE, tenantId.getId()); + }, pageLink, entityId.getEntityType(), entityId.getId(), eventType, tenantId.getId()); } + + protected JsonNode getMetadata(Event outEvent) { String metaDataStr = outEvent.getBody().get("metadata").asText(); try { diff --git a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java deleted file mode 100644 index 9dcba19044..0000000000 --- a/application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.controller; - -import org.junit.BeforeClass; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; -import org.thingsboard.server.queue.memory.InMemoryStorage; - -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({ -// "org.thingsboard.server.controller.sql.WebsocketApiSqlTest", -// "org.thingsboard.server.controller.sql.EntityQueryControllerSqlTest", -// "org.thingsboard.server.controller.sql.TbResourceControllerSqlTest", -// "org.thingsboard.server.controller.sql.DeviceProfileControllerSqlTest", - "org.thingsboard.server.controller.sql.*Test", -}) -public class ControllerSqlTestSuite { - -} diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 9f82b6ca2b..5d86e6a2a5 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -15,30 +15,28 @@ */ package org.thingsboard.server.rules.lifecycle; -import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.test.context.support.TestPropertySourceUtils; +import org.testcontainers.containers.GenericContainer; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Event; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; @@ -46,16 +44,13 @@ import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.controller.AbstractRuleEngineControllerTest; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.queue.memory.InMemoryStorage; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.awaitility.Awaitility.await; -import static org.mockito.Mockito.spy; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static java.util.concurrent.TimeUnit.MILLISECONDS; /** * @author Valerii Sosliuk @@ -63,9 +58,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j public abstract class AbstractRuleEngineLifecycleIntegrationTest extends AbstractRuleEngineControllerTest { - protected Tenant savedTenant; - protected User tenantAdmin; - @Autowired protected ActorSystemContext actorSystem; @@ -75,50 +67,14 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac @Autowired protected EventService eventService; - @Autowired - protected InMemoryStorage storage; - @Before public void beforeTest() throws Exception { - - EventService spyEventService = spy(eventService); - - Mockito.doAnswer((Answer>) invocation -> { - Object[] args = invocation.getArguments(); - Event event = (Event) args[0]; - ListenableFuture future = eventService.saveAsync(event); - try { - future.get(); - } catch (Exception e) {} - return future; - }).when(spyEventService).saveAsync(Mockito.any(Event.class)); - - ReflectionTestUtils.setField(actorSystem, "eventService", spyEventService); - - loginSysAdmin(); - - Tenant tenant = new Tenant(); - tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); - Assert.assertNotNull(savedTenant); - ruleChainService.deleteRuleChainsByTenantId(savedTenant.getId()); - - tenantAdmin = new User(); - tenantAdmin.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin.setTenantId(savedTenant.getId()); - tenantAdmin.setEmail("tenant2@thingsboard.org"); - tenantAdmin.setFirstName("Joe"); - tenantAdmin.setLastName("Downs"); - - createUserAndLogin(tenantAdmin, "testPassword1"); + loginTenantAdmin(); + ruleChainService.deleteRuleChainsByTenantId(tenantId); } @After public void afterTest() throws Exception { - loginSysAdmin(); - if (savedTenant != null) { - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()).andExpect(status().isOk()); - } } @Test @@ -126,7 +82,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac // Creating Rule Chain RuleChain ruleChain = new RuleChain(); ruleChain.setName("Simple Rule Chain"); - ruleChain.setTenantId(savedTenant.getId()); + ruleChain.setTenantId(tenantId); ruleChain.setRoot(true); ruleChain.setDebugMode(true); ruleChain = saveRuleChain(ruleChain); @@ -149,8 +105,24 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac metaData = saveRuleChainMetaData(metaData); Assert.assertNotNull(metaData); - ruleChain = getRuleChain(ruleChain.getId()); - Assert.assertNotNull(ruleChain.getFirstRuleNodeId()); + final RuleChain ruleChainFinal = getRuleChain(ruleChain.getId()); + Assert.assertNotNull(ruleChainFinal.getFirstRuleNodeId()); + + //TODO find out why RULE_NODE update event did not appear all the time + List rcEvents = Awaitility.await("Rule Node started successfully") + .pollInterval(10, MILLISECONDS) + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> { + List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), DataConstants.LC_EVENT, 1000) + .getData().stream().filter(e -> { + var body = e.getBody(); + return body.has("event") && body.get("event").asText().equals("STARTED") + && body.has("success") && body.get("success").asBoolean(); + }).collect(Collectors.toList()); + debugEvents.forEach((e) -> log.trace("event: {}", e)); + return debugEvents; + }, + x -> x.size() == 1); // Saving the device Device device = new Device(); @@ -158,35 +130,46 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac device.setType("default"); device = doPost("/api/device", device, Device.class); + log.warn("before update attr"); attributesService.save(device.getTenantId(), device.getId(), DataConstants.SERVER_SCOPE, - Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey", "serverAttributeValue"), System.currentTimeMillis()))); - - await("total inMemory queue lag is empty").atMost(30, TimeUnit.SECONDS) - .until(() -> storage.getLagTotal() == 0); - Thread.sleep(1000); - + Collections.singletonList(new BaseAttributeKvEntry(new StringDataEntry("serverAttributeKey", "serverAttributeValue"), System.currentTimeMillis()))) + .get(TIMEOUT, TimeUnit.SECONDS); + log.warn("attr updated"); TbMsgCallback tbMsgCallback = Mockito.mock(TbMsgCallback.class); Mockito.when(tbMsgCallback.isMsgValid()).thenReturn(true); TbMsg tbMsg = TbMsg.newMsg("CUSTOM", device.getId(), new TbMsgMetaData(), "{}", tbMsgCallback); - QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(savedTenant.getId(), tbMsg, null, null); + QueueToRuleEngineMsg qMsg = new QueueToRuleEngineMsg(tenantId, tbMsg, null, null); // Pushing Message to the system + log.warn("before tell tbMsgCallback"); actorSystem.tell(qMsg); - Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); - - - PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); - List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); - - Assert.assertEquals(2, events.size()); + log.warn("awaiting tbMsgCallback"); + Mockito.verify(tbMsgCallback, Mockito.timeout(TimeUnit.SECONDS.toMillis(TIMEOUT))).onSuccess(); + log.warn("awaiting events"); + List events = Awaitility.await("get debug by custom event") + .pollInterval(10, MILLISECONDS) + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> { + List debugEvents = getDebugEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), 1000) + .getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); + log.warn("filtered debug events [{}]", debugEvents.size()); + debugEvents.forEach((e) -> log.warn("event: {}", e)); + return debugEvents; + }, + x -> x.size() == 2); + log.warn("asserting.."); Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); - Assert.assertEquals(ruleChain.getFirstRuleNodeId(), inEvent.getEntityId()); + Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), inEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); - Assert.assertEquals(ruleChain.getFirstRuleNodeId(), outEvent.getEntityId()); + Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), outEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); + log.warn("OUT event {}", outEvent); + log.warn("OUT event metadata {}", getMetadata(outEvent)); + + Assert.assertNotNull("metadata has ss_serverAttributeKey", getMetadata(outEvent).get("ss_serverAttributeKey")); Assert.assertEquals("serverAttributeValue", getMetadata(outEvent).get("ss_serverAttributeKey").asText()); } diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/sql/RuleEngineLifecycleSqlIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/sql/RuleEngineLifecycleSqlIntegrationTest.java index de7bab4e4f..02205248fe 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/sql/RuleEngineLifecycleSqlIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/sql/RuleEngineLifecycleSqlIntegrationTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.rules.lifecycle.sql; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.rules.flow.AbstractRuleEngineFlowIntegrationTest; import org.thingsboard.server.rules.lifecycle.AbstractRuleEngineLifecycleIntegrationTest; /** diff --git a/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java deleted file mode 100644 index 3f25ded621..0000000000 --- a/application/src/test/java/org/thingsboard/server/transport/TransportSqlTestSuite.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport; - -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; - -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({ - "org.thingsboard.server.transport.*.rpc.*Test", - "org.thingsboard.server.transport.*.telemetry.timeseries.sql.*Test", - "org.thingsboard.server.transport.*.telemetry.attributes.*Test", - "org.thingsboard.server.transport.*.attributes.updates.*Test", - "org.thingsboard.server.transport.*.attributes.request.*Test", - "org.thingsboard.server.transport.*.claim.*Test", - "org.thingsboard.server.transport.*.provision.*Test", - "org.thingsboard.server.transport.*.credentials.*Test", - "org.thingsboard.server.transport.lwm2m.*.sql.*Test" -}) -public class TransportSqlTestSuite { - -} diff --git a/common/cache/pom.xml b/common/cache/pom.xml index 47834cb6bb..b52984e0fc 100644 --- a/common/cache/pom.xml +++ b/common/cache/pom.xml @@ -56,6 +56,10 @@ com.github.ben-manes.caffeine caffeine + + javax.annotation + javax.annotation-api + org.apache.commons commons-lang3 diff --git a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java b/common/cache/src/main/java/org/thingsboard/server/cache/CacheSpecsMap.java similarity index 59% rename from application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java rename to common/cache/src/main/java/org/thingsboard/server/cache/CacheSpecsMap.java index 714ad67519..594bfff146 100644 --- a/application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CacheSpecsMap.java @@ -13,20 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.system; +package org.thingsboard.server.cache; -import org.junit.BeforeClass; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; -import org.thingsboard.server.queue.memory.InMemoryStorage; +import lombok.Data; +import lombok.Getter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; -/** - * Created by Valerii Sosliuk on 6/27/2017. - */ -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({ - "org.thingsboard.server.system.sql.*SqlTest", -}) -public class SystemSqlTestSuite { +import java.util.Map; + +@Configuration +@ConfigurationProperties(prefix = "cache") +@Data +public class CacheSpecsMap { + + @Getter + private Map specs; } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java new file mode 100644 index 0000000000..00231ad674 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbCacheTransaction.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Slf4j +@RequiredArgsConstructor +public class CaffeineTbCacheTransaction implements TbCacheTransaction { + @Getter + private final UUID id = UUID.randomUUID(); + private final CaffeineTbTransactionalCache cache; + @Getter + private final List keys; + @Getter + @Setter + private boolean failed; + + private final Map pendingPuts = new LinkedHashMap<>(); + + @Override + public void putIfAbsent(K key, V value) { + pendingPuts.put(key, value); + } + + @Override + public boolean commit() { + return cache.commit(id, pendingPuts); + } + + @Override + public void rollback() { + cache.rollback(id); + } + + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java new file mode 100644 index 0000000000..a18402493c --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineTbTransactionalCache.java @@ -0,0 +1,193 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.cache.CacheManager; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@RequiredArgsConstructor +public abstract class CaffeineTbTransactionalCache implements TbTransactionalCache { + + private final CacheManager cacheManager; + @Getter + private final String cacheName; + + private final Lock lock = new ReentrantLock(); + private final Map> objectTransactions = new HashMap<>(); + private final Map> transactions = new HashMap<>(); + + @Override + public TbCacheValueWrapper get(K key) { + return SimpleTbCacheValueWrapper.wrap(cacheManager.getCache(cacheName).get(key)); + } + + @Override + public void put(K key, V value) { + lock.lock(); + try { + failAllTransactionsByKey(key); + cacheManager.getCache(cacheName).put(key, value); + } finally { + lock.unlock(); + } + } + + @Override + public void putIfAbsent(K key, V value) { + lock.lock(); + try { + failAllTransactionsByKey(key); + doPutIfAbsent(key, value); + } finally { + lock.unlock(); + } + } + + @Override + public void evict(K key) { + lock.lock(); + try { + failAllTransactionsByKey(key); + doEvict(key); + } finally { + lock.unlock(); + } + } + + @Override + public void evict(Collection keys) { + lock.lock(); + try { + keys.forEach(key -> { + failAllTransactionsByKey(key); + doEvict(key); + }); + } finally { + lock.unlock(); + } + } + + @Override + public void evictOrPut(K key, V value) { + //No need to put the value in case of Caffeine, because evict will cancel concurrent transaction used to "get" the missing value from cache. + evict(key); + } + + @Override + public TbCacheTransaction newTransactionForKey(K key) { + return newTransaction(Collections.singletonList(key)); + } + + @Override + public TbCacheTransaction newTransactionForKeys(List keys) { + return newTransaction(keys); + } + + void doPutIfAbsent(Object key, Object value) { + cacheManager.getCache(cacheName).putIfAbsent(key, value); + } + + void doEvict(K key) { + cacheManager.getCache(cacheName).evict(key); + } + + TbCacheTransaction newTransaction(List keys) { + lock.lock(); + try { + var transaction = new CaffeineTbCacheTransaction<>(this, keys); + var transactionId = transaction.getId(); + for (K key : keys) { + objectTransactions.computeIfAbsent(key, k -> new HashSet<>()).add(transactionId); + } + transactions.put(transactionId, transaction); + return transaction; + } finally { + lock.unlock(); + } + } + + public boolean commit(UUID trId, Map pendingPuts) { + lock.lock(); + try { + var tr = transactions.get(trId); + var success = !tr.isFailed(); + if (success) { + for (K key : tr.getKeys()) { + Set otherTransactions = objectTransactions.get(key); + if (otherTransactions != null) { + for (UUID otherTrId : otherTransactions) { + if (trId == null || !trId.equals(otherTrId)) { + transactions.get(otherTrId).setFailed(true); + } + } + } + } + pendingPuts.forEach(this::doPutIfAbsent); + } + removeTransaction(trId); + return success; + } finally { + lock.unlock(); + } + } + + void rollback(UUID id) { + lock.lock(); + try { + removeTransaction(id); + } finally { + lock.unlock(); + } + } + + private void removeTransaction(UUID id) { + CaffeineTbCacheTransaction transaction = transactions.remove(id); + if (transaction != null) { + for (var key : transaction.getKeys()) { + Set transactions = objectTransactions.get(key); + if (transactions != null) { + transactions.remove(id); + if (transactions.isEmpty()) { + objectTransactions.remove(key); + } + } + } + } + } + + private void failAllTransactionsByKey(K key) { + Set transactionsIds = objectTransactions.get(key); + if (transactionsIds != null) { + for (UUID otherTrId : transactionsIds) { + transactions.get(otherTrId).setFailed(true); + } + } + } + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java new file mode 100644 index 0000000000..bd4ac13543 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbCacheTransaction.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisStringCommands; + +import java.io.Serializable; +import java.util.Objects; + +@Slf4j +@RequiredArgsConstructor +public class RedisTbCacheTransaction implements TbCacheTransaction { + + private final RedisTbTransactionalCache cache; + private final RedisConnection connection; + + @Override + public void putIfAbsent(K key, V value) { + cache.put(connection, key, value, RedisStringCommands.SetOption.UPSERT); + } + + @Override + public boolean commit() { + try { + var execResult = connection.exec(); + var result = execResult != null && execResult.stream().anyMatch(Objects::nonNull); + return result; + } finally { + connection.close(); + } + } + + @Override + public void rollback() { + try { + connection.discard(); + } finally { + connection.close(); + } + } + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java new file mode 100644 index 0000000000..ed39bf3716 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -0,0 +1,180 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.support.NullValue; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.RedisStringCommands; +import org.springframework.data.redis.core.types.Expiration; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.StringRedisSerializer; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class RedisTbTransactionalCache implements TbTransactionalCache { + + private static final byte[] BINARY_NULL_VALUE = RedisSerializer.java().serialize(NullValue.INSTANCE); + + @Getter + private final String cacheName; + private final RedisConnectionFactory connectionFactory; + private final RedisSerializer keySerializer = new StringRedisSerializer(); + private final RedisSerializer valueSerializer; + private final Expiration evictExpiration; + private final Expiration cacheTtl; + + public RedisTbTransactionalCache(String cacheName, + CacheSpecsMap cacheSpecsMap, + RedisConnectionFactory connectionFactory, + TBRedisCacheConfiguration configuration, + RedisSerializer valueSerializer) { + this.cacheName = cacheName; + this.connectionFactory = connectionFactory; + this.valueSerializer = valueSerializer; + this.evictExpiration = Expiration.from(configuration.getEvictTtlInMs(), TimeUnit.MILLISECONDS); + if (cacheSpecsMap.getSpecs() != null && cacheSpecsMap.getSpecs().get(cacheName) != null) { + CacheSpecs cacheSpecs = cacheSpecsMap.getSpecs().get(cacheName); + this.cacheTtl = Expiration.from(cacheSpecs.getTimeToLiveInMinutes(), TimeUnit.MINUTES); + } else { + this.cacheTtl = Expiration.persistent(); + } + } + + @Override + public TbCacheValueWrapper get(K key) { + try (var connection = connectionFactory.getConnection()) { + byte[] rawKey = getRawKey(key); + byte[] rawValue = connection.get(rawKey); + if (rawValue == null) { + return null; + } else if (Arrays.equals(rawValue, BINARY_NULL_VALUE)) { + return SimpleTbCacheValueWrapper.empty(); + } else { + V value = valueSerializer.deserialize(rawValue); + return SimpleTbCacheValueWrapper.wrap(value); + } + } + } + + @Override + public void put(K key, V value) { + try (var connection = connectionFactory.getConnection()) { + put(connection, key, value, RedisStringCommands.SetOption.UPSERT); + } + } + + @Override + public void putIfAbsent(K key, V value) { + try (var connection = connectionFactory.getConnection()) { + put(connection, key, value, RedisStringCommands.SetOption.SET_IF_ABSENT); + } + } + + @Override + public void evict(K key) { + try (var connection = connectionFactory.getConnection()) { + connection.del(getRawKey(key)); + } + } + + @Override + public void evict(Collection keys) { + try (var connection = connectionFactory.getConnection()) { + connection.del(keys.stream().map(this::getRawKey).toArray(byte[][]::new)); + } + } + + @Override + public void evictOrPut(K key, V value) { + try (var connection = connectionFactory.getConnection()) { + var rawKey = getRawKey(key); + var records = connection.del(rawKey); + if (records == null || records == 0) { + //We need to put the value in case of Redis, because evict will NOT cancel concurrent transaction used to "get" the missing value from cache. + connection.set(rawKey, getRawValue(value), evictExpiration, RedisStringCommands.SetOption.UPSERT); + } + } + } + + @Override + public TbCacheTransaction newTransactionForKey(K key) { + byte[][] rawKey = new byte[][]{getRawKey(key)}; + RedisConnection connection = watch(rawKey); + return new RedisTbCacheTransaction<>(this, connection); + } + + @Override + public TbCacheTransaction newTransactionForKeys(List keys) { + RedisConnection connection = watch(keys.stream().map(this::getRawKey).toArray(byte[][]::new)); + return new RedisTbCacheTransaction<>(this, connection); + } + + private RedisConnection watch(byte[][] rawKeysList) { + var connection = connectionFactory.getConnection(); + try { + connection.watch(rawKeysList); + connection.multi(); + } catch (Exception e) { + connection.close(); + throw e; + } + return connection; + } + + private byte[] getRawKey(K key) { + String keyString = cacheName + key.toString(); + byte[] rawKey; + try { + rawKey = keySerializer.serialize(keyString); + } catch (Exception e) { + log.warn("Failed to serialize the cache key: {}", key, e); + throw new RuntimeException(e); + } + if (rawKey == null) { + log.warn("Failed to serialize the cache key: {}", key); + throw new IllegalArgumentException("Failed to serialize the cache key!"); + } + return rawKey; + } + + private byte[] getRawValue(V value) { + if (value == null) { + return BINARY_NULL_VALUE; + } else { + try { + return valueSerializer.serialize(value); + } catch (Exception e) { + log.warn("Failed to serialize the cache value: {}", value, e); + throw new RuntimeException(e); + } + } + } + + public void put(RedisConnection connection, K key, V value, RedisStringCommands.SetOption setOption) { + byte[] rawKey = getRawKey(key); + byte[] rawValue = getRawValue(value); + connection.set(rawKey, rawValue, cacheTtl, setOption); + } + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java b/common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java new file mode 100644 index 0000000000..f5607fb760 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import org.springframework.cache.Cache; + +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class SimpleTbCacheValueWrapper implements TbCacheValueWrapper { + + private final T value; + + @Override + public T get() { + return value; + } + + public static SimpleTbCacheValueWrapper empty() { + return new SimpleTbCacheValueWrapper<>(null); + } + + public static SimpleTbCacheValueWrapper wrap(T value) { + return new SimpleTbCacheValueWrapper<>(value); + } + + @SuppressWarnings("unchecked") + public static SimpleTbCacheValueWrapper wrap(Cache.ValueWrapper source) { + return source == null ? null : new SimpleTbCacheValueWrapper<>((T) source.get()); + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java index ebf031c562..9c7071868b 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisCacheConfiguration.java @@ -36,42 +36,45 @@ import redis.clients.jedis.JedisPoolConfig; import java.time.Duration; @Configuration -@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis", matchIfMissing = false) +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @EnableCaching @Data public abstract class TBRedisCacheConfiguration { - @Value("${redis.pool_config.maxTotal}") + @Value("${redis.evictTtlInMs:60000}") + private int evictTtlInMs; + + @Value("${redis.pool_config.maxTotal:128}") private int maxTotal; - @Value("${redis.pool_config.maxIdle}") + @Value("${redis.pool_config.maxIdle:128}") private int maxIdle; - @Value("${redis.pool_config.minIdle}") + @Value("${redis.pool_config.minIdle:16}") private int minIdle; - @Value("${redis.pool_config.testOnBorrow}") + @Value("${redis.pool_config.testOnBorrow:true}") private boolean testOnBorrow; - @Value("${redis.pool_config.testOnReturn}") + @Value("${redis.pool_config.testOnReturn:true}") private boolean testOnReturn; - @Value("${redis.pool_config.testWhileIdle}") + @Value("${redis.pool_config.testWhileIdle:true}") private boolean testWhileIdle; - @Value("${redis.pool_config.minEvictableMs}") + @Value("${redis.pool_config.minEvictableMs:60000}") private long minEvictableMs; - @Value("${redis.pool_config.evictionRunsMs}") + @Value("${redis.pool_config.evictionRunsMs:30000}") private long evictionRunsMs; - @Value("${redis.pool_config.maxWaitMills}") + @Value("${redis.pool_config.maxWaitMills:60000}") private long maxWaitMills; - @Value("${redis.pool_config.numberTestsPerEvictionRun}") + @Value("${redis.pool_config.numberTestsPerEvictionRun:3}") private int numberTestsPerEvictionRun; - @Value("${redis.pool_config.blockWhenExhausted}") + @Value("${redis.pool_config.blockWhenExhausted:true}") private boolean blockWhenExhausted; @Bean diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java index 3538b6e0d7..2cf67e35aa 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisClusterConfiguration.java @@ -29,23 +29,23 @@ import java.util.Collections; import java.util.List; @Configuration -@ConditionalOnMissingBean(CaffeineCacheConfiguration.class) +@ConditionalOnMissingBean(TbCaffeineCacheConfiguration.class) @ConditionalOnProperty(prefix = "redis.connection", value = "type", havingValue = "cluster") public class TBRedisClusterConfiguration extends TBRedisCacheConfiguration { private static final String COMMA = ","; private static final String COLON = ":"; - @Value("${redis.cluster.nodes}") + @Value("${redis.cluster.nodes:}") private String clusterNodes; - @Value("${redis.cluster.max-redirects}") + @Value("${redis.cluster.max-redirects:12}") private Integer maxRedirects; - @Value("${redis.cluster.useDefaultPoolConfig}") + @Value("${redis.cluster.useDefaultPoolConfig:true}") private boolean useDefaultPoolConfig; - @Value("${redis.password}") + @Value("${redis.password:}") private String password; public JedisConnectionFactory loadFactory() { diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisStandaloneConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisStandaloneConfiguration.java index 6a5b6e15e6..1c83f529ce 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisStandaloneConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TBRedisStandaloneConfiguration.java @@ -26,35 +26,35 @@ import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; import java.time.Duration; @Configuration -@ConditionalOnMissingBean(CaffeineCacheConfiguration.class) +@ConditionalOnMissingBean(TbCaffeineCacheConfiguration.class) @ConditionalOnProperty(prefix = "redis.connection", value = "type", havingValue = "standalone") public class TBRedisStandaloneConfiguration extends TBRedisCacheConfiguration { - @Value("${redis.standalone.host}") + @Value("${redis.standalone.host:localhost}") private String host; - @Value("${redis.standalone.port}") + @Value("${redis.standalone.port:6379}") private Integer port; - @Value("${redis.standalone.clientName}") + @Value("${redis.standalone.clientName:standalone}") private String clientName; - @Value("${redis.standalone.connectTimeout}") + @Value("${redis.standalone.connectTimeout:30000}") private Long connectTimeout; - @Value("${redis.standalone.readTimeout}") + @Value("${redis.standalone.readTimeout:60000}") private Long readTimeout; - @Value("${redis.standalone.useDefaultClientConfig}") + @Value("${redis.standalone.useDefaultClientConfig:true}") private boolean useDefaultClientConfig; - @Value("${redis.standalone.usePoolConfig}") + @Value("${redis.standalone.usePoolConfig:false}") private boolean usePoolConfig; - @Value("${redis.db}") + @Value("${redis.db:0}") private Integer db; - @Value("${redis.password}") + @Value("${redis.password:}") private String password; public JedisConnectionFactory loadFactory() { diff --git a/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbCacheTransaction.java similarity index 65% rename from dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java rename to common/cache/src/main/java/org/thingsboard/server/cache/TbCacheTransaction.java index fccbc705b2..210dab139f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbCacheTransaction.java @@ -13,16 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao; +package org.thingsboard.server.cache; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; -import org.junit.runner.RunWith; +public interface TbCacheTransaction { -@RunWith(ClasspathSuite.class) -@ClassnameFilters({ - "org.thingsboard.server.dao.sql.*Test" -}) -public class JpaDaoTestSuite { + void putIfAbsent(K key, V value); + + boolean commit(); + + void rollback(); } diff --git a/application/src/test/java/org/thingsboard/server/edge/EdgeSqlTestSuite.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbCacheValueWrapper.java similarity index 63% rename from application/src/test/java/org/thingsboard/server/edge/EdgeSqlTestSuite.java rename to common/cache/src/main/java/org/thingsboard/server/cache/TbCacheValueWrapper.java index e7fff461f3..66d7a64f15 100644 --- a/application/src/test/java/org/thingsboard/server/edge/EdgeSqlTestSuite.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbCacheValueWrapper.java @@ -13,17 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edge; +package org.thingsboard.server.cache; -import org.junit.BeforeClass; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; -import org.thingsboard.server.queue.memory.InMemoryStorage; +public interface TbCacheValueWrapper { -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({ - "org.thingsboard.server.edge.sql.*Test", -}) -public class EdgeSqlTestSuite { + T get(); } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineCacheConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java similarity index 89% rename from common/cache/src/main/java/org/thingsboard/server/cache/CaffeineCacheConfiguration.java rename to common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java index 373b730d41..664eaa1fb4 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/CaffeineCacheConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java @@ -19,10 +19,8 @@ import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.Ticker; import com.github.benmanes.caffeine.cache.Weigher; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; import org.springframework.cache.caffeine.CaffeineCache; @@ -34,20 +32,20 @@ import org.springframework.context.annotation.Configuration; import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Configuration @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) -@ConfigurationProperties(prefix = "caffeine") @EnableCaching -@Data @Slf4j -public class CaffeineCacheConfiguration { +public class TbCaffeineCacheConfiguration { - private Map specs; + private final CacheSpecsMap configuration; + public TbCaffeineCacheConfiguration(CacheSpecsMap configuration) { + this.configuration = configuration; + } /** * Transaction aware CaffeineCache implementation with TransactionAwareCacheManagerProxy @@ -55,11 +53,11 @@ public class CaffeineCacheConfiguration { */ @Bean public CacheManager cacheManager() { - log.trace("Initializing cache: {} specs {}", Arrays.toString(RemovalCause.values()), specs); + log.trace("Initializing cache: {} specs {}", Arrays.toString(RemovalCause.values()), configuration.getSpecs()); SimpleCacheManager manager = new SimpleCacheManager(); - if (specs != null) { + if (configuration.getSpecs() != null) { List caches = - specs.entrySet().stream() + configuration.getSpecs().entrySet().stream() .map(entry -> buildCache(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); @@ -95,4 +93,5 @@ public class CaffeineCacheConfiguration { return 1; }; } + } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbRedisSerializer.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbRedisSerializer.java new file mode 100644 index 0000000000..574038ea91 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbRedisSerializer.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; + +public class TbRedisSerializer implements RedisSerializer { + + private final RedisSerializer java = RedisSerializer.java(); + + @Override + public byte[] serialize(T t) throws SerializationException { + return java.serialize(t); + } + + @Override + public T deserialize(byte[] bytes) throws SerializationException { + return (T) java.deserialize(bytes); + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java new file mode 100644 index 0000000000..92ab0489c7 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import java.io.Serializable; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +public interface TbTransactionalCache { + + String getCacheName(); + + TbCacheValueWrapper get(K key); + + void put(K key, V value); + + void putIfAbsent(K key, V value); + + void evict(K key); + + void evict(Collection keys); + + void evictOrPut(K key, V value); + + TbCacheTransaction newTransactionForKey(K key); + + TbCacheTransaction newTransactionForKeys(List keys); + + default V getAndPutInTransaction(K key, Supplier dbCall, boolean cacheNullValue) { + TbCacheValueWrapper cacheValueWrapper = get(key); + if (cacheValueWrapper != null) { + return cacheValueWrapper.get(); + } + var cacheTransaction = newTransactionForKey(key); + try { + V dbValue = dbCall.get(); + if (dbValue != null || cacheNullValue) { + cacheTransaction.putIfAbsent(key, dbValue); + cacheTransaction.commit(); + return dbValue; + } else { + cacheTransaction.rollback(); + return null; + } + } catch (Throwable e) { + cacheTransaction.rollback(); + throw e; + } + } + + default R getAndPutInTransaction(K key, Supplier dbCall, Function cacheValueToResult, Function dbValueToCacheValue, boolean cacheNullValue) { + TbCacheValueWrapper cacheValueWrapper = get(key); + if (cacheValueWrapper != null) { + var cacheValue = cacheValueWrapper.get(); + return cacheValue == null ? null : cacheValueToResult.apply(cacheValue); + } + var cacheTransaction = newTransactionForKey(key); + try { + R dbValue = dbCall.get(); + if (dbValue != null || cacheNullValue) { + cacheTransaction.putIfAbsent(key, dbValueToCacheValue.apply(dbValue)); + cacheTransaction.commit(); + return dbValue; + } else { + cacheTransaction.rollback(); + return null; + } + } catch (Throwable e) { + cacheTransaction.rollback(); + throw e; + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/EntitiesCacheManager.java b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCacheEvictEvent.java similarity index 66% rename from dao/src/main/java/org/thingsboard/server/dao/cache/EntitiesCacheManager.java rename to common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCacheEvictEvent.java index 19842eacc4..d192b97249 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cache/EntitiesCacheManager.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCacheEvictEvent.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.cache; +package org.thingsboard.server.cache.device; +import lombok.Data; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -public interface EntitiesCacheManager { +@Data +public class DeviceCacheEvictEvent { - void removeDeviceFromCacheByName(TenantId tenantId, String name); + private final TenantId tenantId; + private final DeviceId deviceId; + private final String newName; + private final String oldName; - void removeDeviceFromCacheById(TenantId tenantId, DeviceId deviceId); - - void removeAssetFromCacheByName(TenantId tenantId, String name); - - void removeEdgeFromCacheByName(TenantId tenantId, String name); } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCacheKey.java b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCacheKey.java new file mode 100644 index 0000000000..77e08a0fd8 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCacheKey.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.device; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; + +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +@Builder +public class DeviceCacheKey implements Serializable { + + private final TenantId tenantId; + private final DeviceId deviceId; + private final String deviceName; + + public DeviceCacheKey(TenantId tenantId, DeviceId deviceId) { + this(tenantId, deviceId, null); + } + + public DeviceCacheKey(TenantId tenantId, String deviceName) { + this(tenantId, null, deviceName); + } + + @Override + public String toString() { + if (deviceId != null) { + return tenantId + "_" + deviceId; + } else { + return tenantId + "_n_" + deviceName; + } + } + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCaffeineCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCaffeineCache.java new file mode 100644 index 0000000000..09e97ed205 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.device; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Device; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("DeviceCache") +public class DeviceCaffeineCache extends CaffeineTbTransactionalCache { + + public DeviceCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.DEVICE_CACHE); + } + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceRedisCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceRedisCache.java new file mode 100644 index 0000000000..99577f0039 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.device; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Device; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("DeviceCache") +public class DeviceRedisCache extends RedisTbTransactionalCache { + + public DeviceRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.DEVICE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java index 7ea7b18f43..aaea36abae 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java @@ -20,7 +20,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; -import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_DATA_CACHE; @Service diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/RedisOtaPackageDataCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/RedisOtaPackageDataCache.java index 3117e1fa43..f6ec28d170 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/RedisOtaPackageDataCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/RedisOtaPackageDataCache.java @@ -21,7 +21,6 @@ import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.stereotype.Service; -import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_DATA_CACHE; @Service diff --git a/common/cache/src/test/java/org/thingsboard/server/cache/CaffeineCacheConfigurationTest.java b/common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java similarity index 87% rename from common/cache/src/test/java/org/thingsboard/server/cache/CaffeineCacheConfigurationTest.java rename to common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java index 37bb9c147b..7e65d6a230 100644 --- a/common/cache/src/test/java/org/thingsboard/server/cache/CaffeineCacheConfigurationTest.java +++ b/common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java @@ -30,16 +30,16 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import static org.assertj.core.api.Assertions.assertThat; @ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = CaffeineCacheConfiguration.class) +@ContextConfiguration(classes = {CacheSpecsMap.class, TbCaffeineCacheConfiguration.class}) @EnableConfigurationProperties @TestPropertySource(properties = { "cache.type=caffeine", - "caffeine.specs.relations.timeToLiveInMinutes=1440", - "caffeine.specs.relations.maxSize=0", - "caffeine.specs.devices.timeToLiveInMinutes=60", - "caffeine.specs.devices.maxSize=100"}) + "cache.specs.relations.timeToLiveInMinutes=1440", + "cache.specs.relations.maxSize=0", + "cache.specs.devices.timeToLiveInMinutes=60", + "cache.specs.devices.maxSize=100"}) @Slf4j -public class CaffeineCacheConfigurationTest { +public class CacheSpecsMapTest { @Autowired CacheManager cacheManager; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 5ae8f68eb1..6497778676 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -37,9 +37,9 @@ public interface AttributesService { ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope); - ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes); + ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes); - ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys); + ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys); List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java index e9950447db..f6dae741b1 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeEventService.java @@ -28,5 +28,9 @@ public interface EdgeEventService { PageData findEdgeEvents(TenantId tenantId, EdgeId edgeId, TimePageLink pageLink, boolean withTsUpdate); + /** + * Executes stored procedure to cleanup old edge events. + * @param ttl the ttl for edge events in seconds + */ void cleanupEvents(long ttl); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index 023e757689..dbd641c1c8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -38,8 +38,6 @@ public interface RelationService { EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - ListenableFuture getRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - boolean saveRelation(TenantId tenantId, EntityRelation relation); ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); diff --git a/dao/pom.xml b/dao/pom.xml index 17d982d02e..1528a74869 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -234,12 +234,8 @@ org.apache.maven.plugins maven-surefire-plugin - ${surfire.version} - **/sql/*Test.java - **/sql/*/*DaoTest.java - **/psql/*Test.java **/nosql/*Test.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 18444bb5f3..4f250c47c8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; - import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; diff --git a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java index de695a9035..dc0b838c43 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java @@ -27,7 +27,7 @@ import org.thingsboard.server.dao.util.TbAutoConfiguration; */ @Configuration @TbAutoConfiguration -@ComponentScan("org.thingsboard.server.dao.sql") +@ComponentScan({"org.thingsboard.server.dao.sql", "org.thingsboard.server.dao.attributes", "org.thingsboard.server.dao.cache", "org.thingsboard.server.cache"}) @EnableJpaRepositories("org.thingsboard.server.dao.sql") @EntityScan("org.thingsboard.server.dao.model.sql") @EnableTransactionManagement diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCacheEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCacheEvictEvent.java new file mode 100644 index 0000000000..ce81e047aa --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCacheEvictEvent.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.asset; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@RequiredArgsConstructor +class AssetCacheEvictEvent { + + private final TenantId tenantId; + private final String newName; + private final String oldName; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCacheKey.java new file mode 100644 index 0000000000..c4a6d5b7fb --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCacheKey.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.asset; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; + +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +@Builder +public class AssetCacheKey implements Serializable { + + private static final long serialVersionUID = 4196610233744512673L; + + private final TenantId tenantId; + private final String name; + + @Override + public String toString() { + return tenantId + "_" + name; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCaffeineCache.java new file mode 100644 index 0000000000..7d9aa706d0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.asset; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("AssetCache") +public class AssetCaffeineCache extends CaffeineTbTransactionalCache { + + public AssetCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.ASSET_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index e7533cc736..0432e69bca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetRedisCache.java new file mode 100644 index 0000000000..9a8526f3cf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.asset; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("AssetCache") +public class AssetRedisCache extends RedisTbTransactionalCache { + + public AssetRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index bf8d75d27a..729c923e54 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -22,12 +22,14 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.annotation.CacheEvict; -import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; @@ -42,8 +44,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.dao.cache.EntitiesCacheManager; -import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -55,7 +56,6 @@ import java.util.List; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.CacheConstants.ASSET_CACHE; import static org.thingsboard.server.dao.DaoUtil.toUUIDs; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validateIds; @@ -64,7 +64,7 @@ import static org.thingsboard.server.dao.service.Validator.validateString; @Service @Slf4j -public class BaseAssetService extends AbstractEntityService implements AssetService { +public class BaseAssetService extends AbstractCachedEntityService implements AssetService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_CUSTOMER_ID = "Incorrect customerId "; @@ -74,12 +74,20 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ @Autowired private AssetDao assetDao; - @Autowired - private EntitiesCacheManager cacheManager; - @Autowired private DataValidator assetValidator; + @TransactionalEventListener(classes = AssetCacheEvictEvent.class) + @Override + public void handleEvictEvent(AssetCacheEvictEvent event) { + List keys = new ArrayList<>(2); + keys.add(new AssetCacheKey(event.getTenantId(), event.getNewName())); + if (StringUtils.isNotEmpty(event.getOldName()) && !event.getOldName().equals(event.getNewName())) { + keys.add(new AssetCacheKey(event.getTenantId(), event.getOldName())); + } + cache.evict(keys); + } + @Override public AssetInfo findAssetInfoById(TenantId tenantId, AssetId assetId) { log.trace("Executing findAssetInfoById [{}]", assetId); @@ -101,24 +109,26 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ return assetDao.findByIdAsync(tenantId, assetId.getId()); } - @Cacheable(cacheNames = ASSET_CACHE, key = "{#tenantId, #name}") @Override public Asset findAssetByTenantIdAndName(TenantId tenantId, String name) { log.trace("Executing findAssetByTenantIdAndName [{}][{}]", tenantId, name); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - return assetDao.findAssetsByTenantIdAndName(tenantId.getId(), name) - .orElse(null); + return cache.getAndPutInTransaction(new AssetCacheKey(tenantId, name), + () -> assetDao.findAssetsByTenantIdAndName(tenantId.getId(), name) + .orElse(null), true); } - @CacheEvict(cacheNames = ASSET_CACHE, key = "{#asset.tenantId, #asset.name}") @Override public Asset saveAsset(Asset asset) { log.trace("Executing saveAsset [{}]", asset); - assetValidator.validate(asset, Asset::getTenantId); + Asset oldAsset = assetValidator.validate(asset, Asset::getTenantId); Asset savedAsset; + AssetCacheEvictEvent evictEvent = new AssetCacheEvictEvent(asset.getTenantId(), asset.getName(), oldAsset != null ? oldAsset.getName() : null); try { savedAsset = assetDao.save(asset.getTenantId(), asset); + publishEvictEvent(evictEvent); } catch (Exception t) { + handleEvictEvent(evictEvent); ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("asset_name_unq_key")) { throw new DataValidationException("Asset with such name already exists!"); @@ -160,7 +170,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ throw new RuntimeException("Exception while finding entity views for assetId [" + assetId + "]", e); } - cacheManager.removeAssetFromCacheByName(asset.getTenantId(), asset.getName()); + publishEvictEvent(new AssetCacheEvictEvent(asset.getTenantId(), asset.getName(), null)); assetDao.removeById(tenantId, assetId.getId()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java new file mode 100644 index 0000000000..5a44f47110 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.attributes; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("AttributeCache") +public class AttributeCaffeineCache extends CaffeineTbTransactionalCache { + + public AttributeCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.ATTRIBUTES_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java new file mode 100644 index 0000000000..80c52105fd --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.attributes; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("AttributeCache") +public class AttributeRedisCache extends RedisTbTransactionalCache { + + public AttributeRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.ATTRIBUTES_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java deleted file mode 100644 index 6d620800ca..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesCacheWrapper.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.attributes; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; - -import static org.thingsboard.server.common.data.CacheConstants.ATTRIBUTES_CACHE; - -@Service -@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "true") -@Primary -@Slf4j -public class AttributesCacheWrapper { - private final Cache attributesCache; - - public AttributesCacheWrapper(CacheManager cacheManager) { - this.attributesCache = cacheManager.getCache(ATTRIBUTES_CACHE); - } - - public Cache.ValueWrapper get(AttributeCacheKey attributeCacheKey) { - try { - return attributesCache.get(attributeCacheKey); - } catch (Exception e) { - log.debug("Failed to retrieve element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage()); - return null; - } - } - - public void put(AttributeCacheKey attributeCacheKey, AttributeKvEntry attributeKvEntry) { - try { - attributesCache.put(attributeCacheKey, attributeKvEntry); - } catch (Exception e) { - log.debug("Failed to put element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage()); - } - } - - public void evict(AttributeCacheKey attributeCacheKey) { - try { - attributesCache.evict(attributeCacheKey); - } catch (Exception e) { - log.debug("Failed to evict element from cache for key {}. Reason - {}.", attributeCacheKey, e.getMessage()); - } - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index e3819489de..9f8d9d815d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -31,15 +31,15 @@ import java.util.Optional; */ public interface AttributesDao { - ListenableFuture> find(TenantId tenantId, EntityId entityId, String attributeType, String attributeKey); + Optional find(TenantId tenantId, EntityId entityId, String attributeType, String attributeKey); - ListenableFuture> find(TenantId tenantId, EntityId entityId, String attributeType, Collection attributeKey); + List find(TenantId tenantId, EntityId entityId, String attributeType, Collection attributeKey); - ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String attributeType); + List findAll(TenantId tenantId, EntityId entityId, String attributeType); - ListenableFuture save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute); + ListenableFuture save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute); - ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List keys); + List> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List keys); List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 8d6c57e709..97c263ad97 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -53,20 +53,20 @@ public class BaseAttributesService implements AttributesService { public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { validate(entityId, scope); Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey); - return attributesDao.find(tenantId, entityId, scope, attributeKey); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKey)); } @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { validate(entityId, scope); attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); - return attributesDao.find(tenantId, entityId, scope, attributeKeys); + return Futures.immediateFuture(attributesDao.find(tenantId, entityId, scope, attributeKeys)); } @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { validate(entityId, scope); - return attributesDao.findAll(tenantId, entityId, scope); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); } @Override @@ -80,17 +80,16 @@ public class BaseAttributesService implements AttributesService { } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { validate(entityId, scope); - attributes.forEach(attribute -> validate(attribute)); - - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + attributes.forEach(AttributeUtils::validate); + List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } @Override - public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { validate(entityId, scope); - return attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); + return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index feb4b0e275..3ce56dfbf3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -17,20 +17,21 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.cache.Cache; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; +import org.thingsboard.server.cache.TbCacheValueWrapper; +import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; @@ -46,7 +47,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Executor; import java.util.stream.Collectors; import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @@ -60,22 +60,22 @@ public class CachedAttributesService implements AttributesService { public static final String LOCAL_CACHE_TYPE = "caffeine"; private final AttributesDao attributesDao; - private final AttributesCacheWrapper cacheWrapper; private final CacheExecutorService cacheExecutorService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; - private Executor cacheExecutor; + private final TbTransactionalCache cache; + private ListeningExecutorService cacheExecutor; @Value("${cache.type}") private String cacheType; public CachedAttributesService(AttributesDao attributesDao, - AttributesCacheWrapper cacheWrapper, StatsFactory statsFactory, - CacheExecutorService cacheExecutorService) { + CacheExecutorService cacheExecutorService, + TbTransactionalCache cache) { this.attributesDao = attributesDao; - this.cacheWrapper = cacheWrapper; this.cacheExecutorService = cacheExecutorService; + this.cache = cache; this.hitCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "hit"); this.missCounter = statsFactory.createDefaultCounter(STATS_NAME, "result", "miss"); @@ -88,37 +88,44 @@ public class CachedAttributesService implements AttributesService { /** * Will return: - * - for the local cache type (cache.type="coffeine"): directExecutor (run callback immediately in the same thread) - * - for the remote cache: dedicated thread pool for the cache IO calls to unblock any caller thread - * */ - Executor getExecutor(String cacheType, CacheExecutorService cacheExecutorService) { + * - for the local cache type (cache.type="coffeine"): directExecutor (run callback immediately in the same thread) + * - for the remote cache: dedicated thread pool for the cache IO calls to unblock any caller thread + */ + ListeningExecutorService getExecutor(String cacheType, CacheExecutorService cacheExecutorService) { if (StringUtils.isEmpty(cacheType) || LOCAL_CACHE_TYPE.equals(cacheType)) { log.info("Going to use directExecutor for the local cache type {}", cacheType); - return MoreExecutors.directExecutor(); + return MoreExecutors.newDirectExecutorService(); } log.info("Going to use cacheExecutorService for the remote cache type {}", cacheType); - return cacheExecutorService; + return cacheExecutorService.executor(); } + @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, String attributeKey) { validate(entityId, scope); Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey); AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, attributeKey); - Cache.ValueWrapper cachedAttributeValue = cacheWrapper.get(attributeCacheKey); + TbCacheValueWrapper cachedAttributeValue = cache.get(attributeCacheKey); if (cachedAttributeValue != null) { hitCounter.increment(); - AttributeKvEntry cachedAttributeKvEntry = (AttributeKvEntry) cachedAttributeValue.get(); + AttributeKvEntry cachedAttributeKvEntry = cachedAttributeValue.get(); return Futures.immediateFuture(Optional.ofNullable(cachedAttributeKvEntry)); } else { missCounter.increment(); - ListenableFuture> result = attributesDao.find(tenantId, entityId, scope, attributeKey); - return Futures.transform(result, foundAttrKvEntry -> { - // TODO: think if it's a good idea to store 'empty' attributes - cacheWrapper.put(attributeCacheKey, foundAttrKvEntry.orElse(null)); - return foundAttrKvEntry; - }, cacheExecutor); + return cacheExecutor.submit(() -> { + var cacheTransaction = cache.newTransactionForKey(attributeCacheKey); + try { + Optional result = attributesDao.find(tenantId, entityId, scope, attributeKey); + cacheTransaction.putIfAbsent(attributeCacheKey, result.orElse(null)); + cacheTransaction.commit(); + return result; + } catch (Throwable e) { + cacheTransaction.rollback(); + throw e; + } + }); } } @@ -127,10 +134,10 @@ public class CachedAttributesService implements AttributesService { validate(entityId, scope); attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); - Map wrappedCachedAttributes = findCachedAttributes(entityId, scope, attributeKeys); + Map> wrappedCachedAttributes = findCachedAttributes(entityId, scope, attributeKeys); List cachedAttributes = wrappedCachedAttributes.values().stream() - .map(wrappedCachedAttribute -> (AttributeKvEntry) wrappedCachedAttribute.get()) + .map(TbCacheValueWrapper::get) .filter(Objects::nonNull) .collect(Collectors.toList()); if (wrappedCachedAttributes.size() == attributeKeys.size()) { @@ -140,15 +147,35 @@ public class CachedAttributesService implements AttributesService { Set notFoundAttributeKeys = new HashSet<>(attributeKeys); notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); - ListenableFuture> result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); - return Futures.transform(result, foundInDbAttributes -> mergeDbAndCacheAttributes(entityId, scope, cachedAttributes, notFoundAttributeKeys, foundInDbAttributes), cacheExecutor); - + List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); + + return cacheExecutor.submit(() -> { + var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); + try { + List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); + for (AttributeKvEntry foundInDbAttribute : result) { + AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); + cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); + notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); + } + for (String key : notFoundAttributeKeys) { + cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); + } + List mergedAttributes = new ArrayList<>(cachedAttributes); + mergedAttributes.addAll(result); + cacheTransaction.commit(); + return mergedAttributes; + } catch (Throwable e) { + cacheTransaction.rollback(); + throw e; + } + }); } - private Map findCachedAttributes(EntityId entityId, String scope, Collection attributeKeys) { - Map cachedAttributes = new HashMap<>(); + private Map> findCachedAttributes(EntityId entityId, String scope, Collection attributeKeys) { + Map> cachedAttributes = new HashMap<>(); for (String attributeKey : attributeKeys) { - Cache.ValueWrapper cachedAttributeValue = cacheWrapper.get(new AttributeCacheKey(scope, entityId, attributeKey)); + var cachedAttributeValue = cache.get(new AttributeCacheKey(scope, entityId, attributeKey)); if (cachedAttributeValue != null) { hitCounter.increment(); cachedAttributes.put(attributeKey, cachedAttributeValue); @@ -159,24 +186,10 @@ public class CachedAttributesService implements AttributesService { return cachedAttributes; } - private List mergeDbAndCacheAttributes(EntityId entityId, String scope, List cachedAttributes, Set notFoundAttributeKeys, List foundInDbAttributes) { - for (AttributeKvEntry foundInDbAttribute : foundInDbAttributes) { - AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); - cacheWrapper.put(attributeCacheKey, foundInDbAttribute); - notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); - } - for (String key : notFoundAttributeKeys){ - cacheWrapper.put(new AttributeCacheKey(scope, entityId, key), null); - } - List mergedAttributes = new ArrayList<>(cachedAttributes); - mergedAttributes.addAll(foundInDbAttributes); - return mergedAttributes; - } - @Override public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String scope) { validate(entityId, scope); - return attributesDao.findAll(tenantId, entityId, scope); + return Futures.immediateFuture(attributesDao.findAll(tenantId, entityId, scope)); } @Override @@ -190,34 +203,30 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { + public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { validate(entityId, scope); attributes.forEach(AttributeUtils::validate); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); - ListenableFuture> future = Futures.allAsList(saveFutures); + List> futures = new ArrayList<>(attributes.size()); + for (var attribute : attributes) { + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + futures.add(Futures.transform(future, key -> { + cache.evictOrPut(new AttributeCacheKey(scope, entityId, key), attribute); + return key; + }, cacheExecutor)); + } - // TODO: can do if (attributesCache.get() != null) attributesCache.put() instead, but will be more twice more requests to cache - List attributeKeys = attributes.stream().map(KvEntry::getKey).collect(Collectors.toList()); - future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), cacheExecutor); - return future; + return Futures.allAsList(futures); } @Override - public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { + public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { validate(entityId, scope); - ListenableFuture> future = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); - future.addListener(() -> evictAttributesFromCache(tenantId, entityId, scope, attributeKeys), cacheExecutor); - return future; + List> futures = attributesDao.removeAll(tenantId, entityId, scope, attributeKeys); + return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, key -> { + cache.evict(new AttributeCacheKey(scope, entityId, key)); + return key; + }, cacheExecutor)).collect(Collectors.toList())); } - private void evictAttributesFromCache(TenantId tenantId, EntityId entityId, String scope, List attributeKeys) { - try { - for (String attributeKey : attributeKeys) { - cacheWrapper.evict(new AttributeCacheKey(scope, entityId, attributeKey)); - } - } catch (Exception e) { - log.error("[{}][{}] Failed to remove values from cache.", tenantId, entityId, e); - } - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/EntitiesCacheManagerImpl.java b/dao/src/main/java/org/thingsboard/server/dao/cache/EntitiesCacheManagerImpl.java deleted file mode 100644 index 2a28f07424..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/cache/EntitiesCacheManagerImpl.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.cache; - -import lombok.AllArgsConstructor; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.TenantId; - -import java.util.Arrays; - -import static org.thingsboard.server.common.data.CacheConstants.ASSET_CACHE; -import static org.thingsboard.server.common.data.CacheConstants.DEVICE_CACHE; -import static org.thingsboard.server.common.data.CacheConstants.EDGE_CACHE; - -@Component -@AllArgsConstructor -public class EntitiesCacheManagerImpl implements EntitiesCacheManager { - - private final CacheManager cacheManager; - - @Override - public void removeDeviceFromCacheByName(TenantId tenantId, String name) { - Cache cache = cacheManager.getCache(DEVICE_CACHE); - cache.evict(Arrays.asList(tenantId, name)); - } - - @Override - public void removeDeviceFromCacheById(TenantId tenantId, DeviceId deviceId) { - if (deviceId == null) { - return; - } - Cache cache = cacheManager.getCache(DEVICE_CACHE); - cache.evict(Arrays.asList(tenantId, deviceId)); - } - - @Override - public void removeAssetFromCacheByName(TenantId tenantId, String name) { - Cache cache = cacheManager.getCache(ASSET_CACHE); - cache.evict(Arrays.asList(tenantId, name)); - } - - @Override - public void removeEdgeFromCacheByName(TenantId tenantId, String name) { - Cache cache = cacheManager.getCache(EDGE_CACHE); - cache.evict(Arrays.asList(tenantId, name)); - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cache/PreviousDeviceCredentialsIdKeyGenerator.java b/dao/src/main/java/org/thingsboard/server/dao/cache/PreviousDeviceCredentialsIdKeyGenerator.java deleted file mode 100644 index 15c560caed..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/cache/PreviousDeviceCredentialsIdKeyGenerator.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.cache; - -import org.springframework.cache.interceptor.KeyGenerator; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.dao.device.DeviceCredentialsService; - -import java.lang.reflect.Method; - -import static org.thingsboard.server.common.data.CacheConstants.DEVICE_CREDENTIALS_CACHE; - -@Component("previousDeviceCredentialsId") -public class PreviousDeviceCredentialsIdKeyGenerator implements KeyGenerator { - - private static final String NOT_VALID_DEVICE = DEVICE_CREDENTIALS_CACHE + "_notValidDeviceCredentialsId"; - - @Override - public Object generate(Object o, Method method, Object... objects) { - DeviceCredentialsService deviceCredentialsService = (DeviceCredentialsService) o; - TenantId tenantId = (TenantId) objects[0]; - DeviceCredentials deviceCredentials = (DeviceCredentials) objects[1]; - if (deviceCredentials.getDeviceId() != null) { - DeviceCredentials oldDeviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, deviceCredentials.getDeviceId()); - if (oldDeviceCredentials != null) { - return DEVICE_CREDENTIALS_CACHE + "_" + oldDeviceCredentials.getCredentialsId(); - } - } - return NOT_VALID_DEVICE; - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 7f247f5ba2..c79ec44880 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.id.CustomerId; @@ -69,6 +70,7 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom @Autowired private DashboardService dashboardService; + @Lazy @Autowired private ApiUsageStateService apiUsageStateService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsCaffeineCache.java new file mode 100644 index 0000000000..1568363275 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("DeviceCredentialsCache") +public class DeviceCredentialsCaffeineCache extends CaffeineTbTransactionalCache { + + public DeviceCredentialsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.DEVICE_CREDENTIALS_CACHE); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/ServiceSqlTestSuite.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsEvictEvent.java similarity index 59% rename from application/src/test/java/org/thingsboard/server/service/ServiceSqlTestSuite.java rename to dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsEvictEvent.java index 11f06875fe..ddf1061b04 100644 --- a/application/src/test/java/org/thingsboard/server/service/ServiceSqlTestSuite.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsEvictEvent.java @@ -13,18 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service; +package org.thingsboard.server.dao.device; -import org.junit.BeforeClass; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; -import org.thingsboard.server.queue.memory.InMemoryStorage; +import lombok.Data; -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({ - "org.thingsboard.server.service.resource.sql.*Test", - "org.thingsboard.server.service.sql.*Test" -}) -public class ServiceSqlTestSuite { +@Data +class DeviceCredentialsEvictEvent { + + private final String newCedentialsId; + private final String oldCredentialsId; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsRedisCache.java new file mode 100644 index 0000000000..1d8a9455dc --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("DeviceCredentialsCache") +public class DeviceCredentialsRedisCache extends RedisTbTransactionalCache { + + public DeviceCredentialsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.DEVICE_CREDENTIALS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java index ff92c330e0..b6ea2b8c4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsServiceImpl.java @@ -15,15 +15,15 @@ */ package org.thingsboard.server.dao.device; - import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.SecurityMode; import org.eclipse.leshan.core.util.SecurityUtil; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.annotation.CacheEvict; -import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; @@ -41,18 +41,17 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.EncryptionUtil; -import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.service.DataValidator; -import static org.thingsboard.server.common.data.CacheConstants.DEVICE_CREDENTIALS_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validateString; @Service @Slf4j -public class DeviceCredentialsServiceImpl extends AbstractEntityService implements DeviceCredentialsService { +public class DeviceCredentialsServiceImpl extends AbstractCachedEntityService implements DeviceCredentialsService { @Autowired private DeviceCredentialsDao deviceCredentialsDao; @@ -60,6 +59,15 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen @Autowired private DataValidator credentialsValidator; + @TransactionalEventListener(classes = DeviceCredentialsEvictEvent.class) + @Override + public void handleEvictEvent(DeviceCredentialsEvictEvent event) { + cache.evict(event.getNewCedentialsId()); + if (StringUtils.isNotEmpty(event.getOldCredentialsId()) && !event.getNewCedentialsId().equals(event.getOldCredentialsId())) { + cache.evict(event.getOldCredentialsId()); + } + } + @Override public DeviceCredentials findDeviceCredentialsByDeviceId(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceCredentialsByDeviceId [{}]", deviceId); @@ -68,15 +76,15 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen } @Override - @Cacheable(cacheNames = DEVICE_CREDENTIALS_CACHE, key = "'deviceCredentials_' + #credentialsId", unless = "#result == null") public DeviceCredentials findDeviceCredentialsByCredentialsId(String credentialsId) { log.trace("Executing findDeviceCredentialsByCredentialsId [{}]", credentialsId); validateString(credentialsId, "Incorrect credentialsId " + credentialsId); - return deviceCredentialsDao.findByCredentialsId(TenantId.SYS_TENANT_ID, credentialsId); + return cache.getAndPutInTransaction(credentialsId, + () -> deviceCredentialsDao.findByCredentialsId(TenantId.SYS_TENANT_ID, credentialsId), + false); } @Override - @CacheEvict(cacheNames = DEVICE_CREDENTIALS_CACHE, keyGenerator = "previousDeviceCredentialsId", beforeInvocation = true) public DeviceCredentials updateDeviceCredentials(TenantId tenantId, DeviceCredentials deviceCredentials) { return saveOrUpdate(tenantId, deviceCredentials); } @@ -93,9 +101,16 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen formatCredentials(deviceCredentials); log.trace("Executing updateDeviceCredentials [{}]", deviceCredentials); credentialsValidator.validate(deviceCredentials, id -> tenantId); + DeviceCredentials oldDeviceCredentials = null; + if (deviceCredentials.getDeviceId() != null) { + oldDeviceCredentials = deviceCredentialsDao.findByDeviceId(tenantId, deviceCredentials.getDeviceId().getId()); + } try { - return deviceCredentialsDao.saveAndFlush(tenantId, deviceCredentials); + var value = deviceCredentialsDao.saveAndFlush(tenantId, deviceCredentials); + publishEvictEvent(new DeviceCredentialsEvictEvent(value.getCredentialsId(), oldDeviceCredentials != null ? oldDeviceCredentials.getCredentialsId() : null)); + return value; } catch (Exception t) { + handleEvictEvent(new DeviceCredentialsEvictEvent(deviceCredentials.getCredentialsId(), oldDeviceCredentials != null ? oldDeviceCredentials.getCredentialsId() : null)); ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && (e.getConstraintName().equalsIgnoreCase("device_credentials_id_unq_key") || e.getConstraintName().equalsIgnoreCase("device_credentials_device_id_unq_key"))) { @@ -183,8 +198,7 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen deviceCredentials.setCredentialsValue(JacksonUtil.toString(lwM2MCredentials)); X509ClientCredential x509ClientConfig = (X509ClientCredential) clientCredentials; if ((StringUtils.isNotBlank(x509ClientConfig.getCert()))) { - String sha3Hash = EncryptionUtil.getSha3Hash(x509ClientConfig.getCert()); - credentialsId = sha3Hash; + credentialsId = EncryptionUtil.getSha3Hash(x509ClientConfig.getCert()); } else { credentialsId = x509ClientConfig.getEndpoint(); } @@ -252,7 +266,7 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen throw new DeviceCredentialsValidationException("LwM2M client PSK key must be random sequence in hex encoding!"); } - if (pskKey.length()% 32 != 0 || pskKey.length() > 128) { + if (pskKey.length() % 32 != 0 || pskKey.length() > 128) { throw new DeviceCredentialsValidationException("LwM2M client PSK key length = " + pskKey.length() + ". Key must be HexDec format: 32, 64, 128 characters!"); } @@ -368,10 +382,10 @@ public class DeviceCredentialsServiceImpl extends AbstractEntityService implemen } @Override - @CacheEvict(cacheNames = DEVICE_CREDENTIALS_CACHE, key = "'deviceCredentials_' + #deviceCredentials.credentialsId") public void deleteDeviceCredentials(TenantId tenantId, DeviceCredentials deviceCredentials) { log.trace("Executing deleteDeviceCredentials [{}]", deviceCredentials); deviceCredentialsDao.removeById(tenantId, deviceCredentials.getUuidId()); + publishEvictEvent(new DeviceCredentialsEvictEvent(deviceCredentials.getCredentialsId(), null)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileCacheKey.java new file mode 100644 index 0000000000..79f997e22c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileCacheKey.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device; + +import lombok.Data; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; + +@Data +public class DeviceProfileCacheKey implements Serializable { + + private static final long serialVersionUID = 8220455917177676472L; + + private final TenantId tenantId; + private final String name; + private final DeviceProfileId deviceProfileId; + private final boolean defaultProfile; + + private DeviceProfileCacheKey(TenantId tenantId, String name, DeviceProfileId deviceProfileId, boolean defaultProfile) { + this.tenantId = tenantId; + this.name = name; + this.deviceProfileId = deviceProfileId; + this.defaultProfile = defaultProfile; + } + + public static DeviceProfileCacheKey fromName(TenantId tenantId, String name) { + return new DeviceProfileCacheKey(tenantId, name, null, false); + } + + public static DeviceProfileCacheKey fromId(DeviceProfileId id) { + return new DeviceProfileCacheKey(null, null, id, false); + } + + public static DeviceProfileCacheKey defaultProfile(TenantId tenantId) { + return new DeviceProfileCacheKey(tenantId, null, null, true); + } + + @Override + public String toString() { + if (deviceProfileId != null) { + return deviceProfileId.toString(); + } else if (defaultProfile) { + return tenantId.toString(); + } else { + return tenantId + "_" + name; + } + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileCaffeineCache.java new file mode 100644 index 0000000000..2e369dec9c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("DeviceProfileCache") +public class DeviceProfileCaffeineCache extends CaffeineTbTransactionalCache { + + public DeviceProfileCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.DEVICE_PROFILE_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileEvictEvent.java new file mode 100644 index 0000000000..b5b4b27a6b --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileEvictEvent.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device; + +import lombok.Data; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class DeviceProfileEvictEvent { + + private final TenantId tenantId; + private final String newName; + private final String oldName; + private final DeviceProfileId deviceProfileId; + private final boolean defaultProfile; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileRedisCache.java new file mode 100644 index 0000000000..f3d7ad634f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.device; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("DeviceProfileCache") +public class DeviceProfileRedisCache extends RedisTbTransactionalCache { + + public DeviceProfileRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.DEVICE_PROFILE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 602128818b..04084de987 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -18,16 +18,17 @@ package org.thingsboard.server.dao.device; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; @@ -36,27 +37,27 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import java.util.Arrays; -import java.util.Collections; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import static org.thingsboard.server.common.data.CacheConstants.DEVICE_PROFILE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @Service @Slf4j -public class DeviceProfileServiceImpl extends AbstractEntityService implements DeviceProfileService { +public class DeviceProfileServiceImpl extends AbstractCachedEntityService implements DeviceProfileService { private static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; private static final String INCORRECT_DEVICE_PROFILE_ID = "Incorrect deviceProfileId "; private static final String INCORRECT_DEVICE_PROFILE_NAME = "Incorrect deviceProfileName "; + private static final String DEVICE_PROFILE_WITH_SUCH_NAME_ALREADY_EXISTS = "Device profile with such name already exists!"; @Autowired private DeviceProfileDao deviceProfileDao; @@ -67,66 +68,73 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D @Autowired private DeviceService deviceService; - @Autowired - private CacheManager cacheManager; - @Autowired private DataValidator deviceProfileValidator; private final Lock findOrCreateLock = new ReentrantLock(); - @Cacheable(cacheNames = DEVICE_PROFILE_CACHE, key = "{#deviceProfileId.id}") + @TransactionalEventListener(classes = DeviceProfileEvictEvent.class) + @Override + public void handleEvictEvent(DeviceProfileEvictEvent event) { + List keys = new ArrayList<>(2); + keys.add(DeviceProfileCacheKey.fromName(event.getTenantId(), event.getNewName())); + if (event.getDeviceProfileId() != null) { + keys.add(DeviceProfileCacheKey.fromId(event.getDeviceProfileId())); + } + if (event.isDefaultProfile()) { + keys.add(DeviceProfileCacheKey.defaultProfile(event.getTenantId())); + } + if (StringUtils.isNotEmpty(event.getOldName()) && !event.getOldName().equals(event.getNewName())) { + keys.add(DeviceProfileCacheKey.fromName(event.getTenantId(), event.getOldName())); + } + cache.evict(keys); + } + @Override public DeviceProfile findDeviceProfileById(TenantId tenantId, DeviceProfileId deviceProfileId) { log.trace("Executing findDeviceProfileById [{}]", deviceProfileId); Validator.validateId(deviceProfileId, INCORRECT_DEVICE_PROFILE_ID + deviceProfileId); - return deviceProfileDao.findById(tenantId, deviceProfileId.getId()); + return cache.getAndPutInTransaction(DeviceProfileCacheKey.fromId(deviceProfileId), + () -> deviceProfileDao.findById(tenantId, deviceProfileId.getId()), true); } @Override public DeviceProfile findDeviceProfileByName(TenantId tenantId, String profileName) { log.trace("Executing findDeviceProfileByName [{}][{}]", tenantId, profileName); Validator.validateString(profileName, INCORRECT_DEVICE_PROFILE_NAME + profileName); - return deviceProfileDao.findByName(tenantId, profileName); + return cache.getAndPutInTransaction(DeviceProfileCacheKey.fromName(tenantId, profileName), + () -> deviceProfileDao.findByName(tenantId, profileName), true); } - @Cacheable(cacheNames = DEVICE_PROFILE_CACHE, key = "{'info', #deviceProfileId.id}") @Override public DeviceProfileInfo findDeviceProfileInfoById(TenantId tenantId, DeviceProfileId deviceProfileId) { log.trace("Executing findDeviceProfileById [{}]", deviceProfileId); Validator.validateId(deviceProfileId, INCORRECT_DEVICE_PROFILE_ID + deviceProfileId); - return deviceProfileDao.findDeviceProfileInfoById(tenantId, deviceProfileId.getId()); + return toDeviceProfileInfo(findDeviceProfileById(tenantId, deviceProfileId)); } @Override public DeviceProfile saveDeviceProfile(DeviceProfile deviceProfile) { log.trace("Executing saveDeviceProfile [{}]", deviceProfile); - deviceProfileValidator.validate(deviceProfile, DeviceProfile::getTenantId); - DeviceProfile oldDeviceProfile = null; - if (deviceProfile.getId() != null) { - oldDeviceProfile = deviceProfileDao.findById(deviceProfile.getTenantId(), deviceProfile.getId().getId()); - } + DeviceProfile oldDeviceProfile = deviceProfileValidator.validate(deviceProfile, DeviceProfile::getTenantId); DeviceProfile savedDeviceProfile; try { savedDeviceProfile = deviceProfileDao.saveAndFlush(deviceProfile.getTenantId(), deviceProfile); + publishEvictEvent(new DeviceProfileEvictEvent(savedDeviceProfile.getTenantId(), savedDeviceProfile.getName(), + oldDeviceProfile != null ? oldDeviceProfile.getName() : null, savedDeviceProfile.getId(), savedDeviceProfile.isDefault())); } catch (Exception t) { + handleEvictEvent(new DeviceProfileEvictEvent(deviceProfile.getTenantId(), deviceProfile.getName(), + oldDeviceProfile != null ? oldDeviceProfile.getName() : null, null, deviceProfile.isDefault())); ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("device_profile_name_unq_key")) { - throw new DataValidationException("Device profile with such name already exists!"); + //TODO: refactor this to return existing device profile. If they are equal - no need to throw exception. Then we can make this call @Transactional and tests will not fail. + throw new DataValidationException(DEVICE_PROFILE_WITH_SUCH_NAME_ALREADY_EXISTS); } else if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("device_provision_key_unq_key")) { throw new DataValidationException("Device profile with such provision device key already exists!"); } else { throw t; } } - Cache cache = cacheManager.getCache(DEVICE_PROFILE_CACHE); - cache.evict(Collections.singletonList(savedDeviceProfile.getId().getId())); - cache.evict(Arrays.asList("info", savedDeviceProfile.getId().getId())); - cache.evict(Arrays.asList(deviceProfile.getTenantId().getId(), deviceProfile.getName())); - if (savedDeviceProfile.isDefault()) { - cache.evict(Arrays.asList("default", savedDeviceProfile.getTenantId().getId())); - cache.evict(Arrays.asList("default", "info", savedDeviceProfile.getTenantId().getId())); - } if (oldDeviceProfile != null && !oldDeviceProfile.getName().equals(deviceProfile.getName())) { PageLink pageLink = new PageLink(100); PageData pageData; @@ -157,6 +165,8 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D DeviceProfileId deviceProfileId = deviceProfile.getId(); try { deviceProfileDao.removeById(tenantId, deviceProfileId.getId()); + publishEvictEvent(new DeviceProfileEvictEvent(deviceProfile.getTenantId(), deviceProfile.getName(), + null, deviceProfile.getId(), deviceProfile.isDefault())); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("fk_device_profile")) { @@ -166,10 +176,6 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D } } deleteEntityRelations(tenantId, deviceProfileId); - Cache cache = cacheManager.getCache(DEVICE_PROFILE_CACHE); - cache.evict(Collections.singletonList(deviceProfileId.getId())); - cache.evict(Arrays.asList("info", deviceProfileId.getId())); - cache.evict(Arrays.asList(tenantId.getId(), deviceProfile.getName())); } @Override @@ -188,20 +194,19 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D return deviceProfileDao.findDeviceProfileInfos(tenantId, pageLink, transportType); } - @Cacheable(cacheNames = DEVICE_PROFILE_CACHE, key = "{#tenantId.id, #name}") @Override public DeviceProfile findOrCreateDeviceProfile(TenantId tenantId, String name) { log.trace("Executing findOrCreateDefaultDeviceProfile"); DeviceProfile deviceProfile = findDeviceProfileByName(tenantId, name); if (deviceProfile == null) { - findOrCreateLock.lock(); try { - deviceProfile = findDeviceProfileByName(tenantId, name); - if (deviceProfile == null) { - deviceProfile = this.doCreateDefaultDeviceProfile(tenantId, name, name.equals("default")); + deviceProfile = this.doCreateDefaultDeviceProfile(tenantId, name, name.equals("default")); + } catch (DataValidationException e) { + if (DEVICE_PROFILE_WITH_SUCH_NAME_ALREADY_EXISTS.equals(e.getMessage())) { + deviceProfile = findDeviceProfileByName(tenantId, name); + } else { + throw e; } - } finally { - findOrCreateLock.unlock(); } } return deviceProfile; @@ -234,20 +239,19 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D return saveDeviceProfile(deviceProfile); } - @Cacheable(cacheNames = DEVICE_PROFILE_CACHE, key = "{'default', #tenantId.id}") @Override public DeviceProfile findDefaultDeviceProfile(TenantId tenantId) { log.trace("Executing findDefaultDeviceProfile tenantId [{}]", tenantId); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - return deviceProfileDao.findDefaultDeviceProfile(tenantId); + return cache.getAndPutInTransaction(DeviceProfileCacheKey.defaultProfile(tenantId), + () -> deviceProfileDao.findDefaultDeviceProfile(tenantId), true); } - @Cacheable(cacheNames = DEVICE_PROFILE_CACHE, key = "{'default', 'info', #tenantId.id}") @Override public DeviceProfileInfo findDefaultDeviceProfileInfo(TenantId tenantId) { log.trace("Executing findDefaultDeviceProfileInfo tenantId [{}]", tenantId); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - return deviceProfileDao.findDefaultDeviceProfileInfo(tenantId); + return toDeviceProfileInfo(findDefaultDeviceProfile(tenantId)); } @Override @@ -256,29 +260,21 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D Validator.validateId(deviceProfileId, INCORRECT_DEVICE_PROFILE_ID + deviceProfileId); DeviceProfile deviceProfile = deviceProfileDao.findById(tenantId, deviceProfileId.getId()); if (!deviceProfile.isDefault()) { - Cache cache = cacheManager.getCache(DEVICE_PROFILE_CACHE); deviceProfile.setDefault(true); DeviceProfile previousDefaultDeviceProfile = findDefaultDeviceProfile(tenantId); boolean changed = false; if (previousDefaultDeviceProfile == null) { deviceProfileDao.save(tenantId, deviceProfile); + publishEvictEvent(new DeviceProfileEvictEvent(deviceProfile.getTenantId(), deviceProfile.getName(), null, deviceProfile.getId(), true)); changed = true; } else if (!previousDefaultDeviceProfile.getId().equals(deviceProfile.getId())) { previousDefaultDeviceProfile.setDefault(false); deviceProfileDao.save(tenantId, previousDefaultDeviceProfile); deviceProfileDao.save(tenantId, deviceProfile); - cache.evict(Collections.singletonList(previousDefaultDeviceProfile.getId().getId())); - cache.evict(Arrays.asList("info", previousDefaultDeviceProfile.getId().getId())); - cache.evict(Arrays.asList(tenantId.getId(), previousDefaultDeviceProfile.getName())); + publishEvictEvent(new DeviceProfileEvictEvent(previousDefaultDeviceProfile.getTenantId(), previousDefaultDeviceProfile.getName(), null, previousDefaultDeviceProfile.getId(), false)); + publishEvictEvent(new DeviceProfileEvictEvent(deviceProfile.getTenantId(), deviceProfile.getName(), null, deviceProfile.getId(), true)); changed = true; } - if (changed) { - cache.evict(Collections.singletonList(deviceProfile.getId().getId())); - cache.evict(Arrays.asList("info", deviceProfile.getId().getId())); - cache.evict(Arrays.asList("default", tenantId.getId())); - cache.evict(Arrays.asList("default", "info", tenantId.getId())); - cache.evict(Arrays.asList(tenantId.getId(), deviceProfile.getName())); - } return changed; } return false; @@ -292,7 +288,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D } private PaginatedRemover tenantDeviceProfilesRemover = - new PaginatedRemover() { + new PaginatedRemover<>() { @Override protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { @@ -305,4 +301,9 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D } }; + private DeviceProfileInfo toDeviceProfileInfo(DeviceProfile profile) { + return profile == null ? null : new DeviceProfileInfo(profile.getId(), profile.getName(), profile.getImage(), + profile.getDefaultDashboardId(), profile.getType(), profile.getTransportType()); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index a3260ffa17..c486473e38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -23,21 +23,22 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.annotation.CacheEvict; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.cache.annotation.Caching; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.device.DeviceCacheEvictEvent; +import org.thingsboard.server.cache.device.DeviceCacheKey; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; import org.thingsboard.server.common.data.device.data.CoapDeviceTransportConfiguration; @@ -62,11 +63,10 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; -import org.thingsboard.server.dao.cache.EntitiesCacheManager; import org.thingsboard.server.dao.device.provision.ProvisionFailedException; import org.thingsboard.server.dao.device.provision.ProvisionRequest; import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; -import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; @@ -77,12 +77,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.CacheConstants.DEVICE_CACHE; import static org.thingsboard.server.dao.DaoUtil.toUUIDs; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validateIds; @@ -91,7 +89,7 @@ import static org.thingsboard.server.dao.service.Validator.validateString; @Service @Slf4j -public class DeviceServiceImpl extends AbstractEntityService implements DeviceService { +public class DeviceServiceImpl extends AbstractCachedEntityService implements DeviceService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_DEVICE_PROFILE_ID = "Incorrect deviceProfileId "; @@ -109,9 +107,6 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe @Autowired private DeviceProfileService deviceProfileService; - @Autowired - private EntitiesCacheManager cacheManager; - @Autowired private EventService eventService; @@ -125,16 +120,19 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe return deviceDao.findDeviceInfoById(tenantId, deviceId.getId()); } - @Cacheable(cacheNames = DEVICE_CACHE, key = "{#tenantId, #deviceId}") @Override public Device findDeviceById(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceById [{}]", deviceId); validateId(deviceId, INCORRECT_DEVICE_ID + deviceId); - if (TenantId.SYS_TENANT_ID.equals(tenantId)) { - return deviceDao.findById(tenantId, deviceId.getId()); - } else { - return deviceDao.findDeviceByTenantIdAndId(tenantId, deviceId.getId()); - } + return cache.getAndPutInTransaction(new DeviceCacheKey(tenantId, deviceId), + () -> { + //TODO: possible bug source since sometimes we need to clear cache by tenant id and sometimes by sys tenant id? + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return deviceDao.findById(tenantId, deviceId.getId()); + } else { + return deviceDao.findDeviceByTenantIdAndId(tenantId, deviceId.getId()); + } + }, true); } @Override @@ -148,47 +146,29 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe } } - @Cacheable(cacheNames = DEVICE_CACHE, key = "{#tenantId, #name}") @Override public Device findDeviceByTenantIdAndName(TenantId tenantId, String name) { log.trace("Executing findDeviceByTenantIdAndName [{}][{}]", tenantId, name); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - Optional deviceOpt = deviceDao.findDeviceByTenantIdAndName(tenantId.getId(), name); - return deviceOpt.orElse(null); + return cache.getAndPutInTransaction(new DeviceCacheKey(tenantId, name), + () -> deviceDao.findDeviceByTenantIdAndName(tenantId.getId(), name).orElse(null), true); } - @Caching(evict= { - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.name}"), - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.id}") - }) - @Transactional @Override public Device saveDeviceWithAccessToken(Device device, String accessToken) { return doSaveDevice(device, accessToken, true); } - @Caching(evict= { - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.name}"), - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.id}") - }) @Override public Device saveDevice(Device device, boolean doValidate) { return doSaveDevice(device, null, doValidate); } - @Caching(evict= { - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.name}"), - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.id}") - }) @Override public Device saveDevice(Device device) { return doSaveDevice(device, null, true); } - @Caching(evict= { - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.name}"), - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.id}") - }) @Transactional @Override public Device saveDeviceWithCredentials(Device device, DeviceCredentials deviceCredentials) { @@ -226,9 +206,13 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe private Device saveDeviceWithoutCredentials(Device device, boolean doValidate) { log.trace("Executing saveDevice [{}]", device); + Device oldDevice = null; if (doValidate) { - deviceValidator.validate(device, Device::getTenantId); + oldDevice = deviceValidator.validate(device, Device::getTenantId); + } else if (device.getId() != null) { + oldDevice = findDeviceById(device.getTenantId(), device.getId()); } + DeviceCacheEvictEvent deviceCacheEvictEvent = new DeviceCacheEvictEvent(device.getTenantId(), device.getId(), device.getName(), oldDevice != null ? oldDevice.getName() : null); try { DeviceProfile deviceProfile; if (device.getDeviceProfileId() == null) { @@ -246,13 +230,13 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe } device.setType(deviceProfile.getName()); device.setDeviceData(syncDeviceData(deviceProfile, device.getDeviceData())); - return deviceDao.saveAndFlush(device.getTenantId(), device); + Device result = deviceDao.saveAndFlush(device.getTenantId(), device); + publishEvictEvent(deviceCacheEvictEvent); + return result; } catch (Exception t) { + handleEvictEvent(deviceCacheEvictEvent); ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("device_name_unq_key")) { - // remove device from cache in case null value cached in the distributed redis. - cacheManager.removeDeviceFromCacheByName(device.getTenantId(), device.getName()); - cacheManager.removeDeviceFromCacheById(device.getTenantId(), device.getId()); throw new DataValidationException("Device with such name already exists!"); } else { throw t; @@ -260,15 +244,27 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe } } + @TransactionalEventListener(classes = DeviceCacheEvictEvent.class) + @Override + public void handleEvictEvent(DeviceCacheEvictEvent event) { + List keys = new ArrayList<>(3); + keys.add(new DeviceCacheKey(event.getTenantId(), event.getNewName())); + if (event.getDeviceId() != null) { + keys.add(new DeviceCacheKey(event.getTenantId(), event.getDeviceId())); + } + if (StringUtils.isNotEmpty(event.getOldName()) && !event.getOldName().equals(event.getNewName())) { + keys.add(new DeviceCacheKey(event.getTenantId(), event.getOldName())); + } + cache.evict(keys); + } + private DeviceData syncDeviceData(DeviceProfile deviceProfile, DeviceData deviceData) { if (deviceData == null) { deviceData = new DeviceData(); } if (deviceData.getConfiguration() == null || !deviceProfile.getType().equals(deviceData.getConfiguration().getType())) { - switch (deviceProfile.getType()) { - case DEFAULT: - deviceData.setConfiguration(new DefaultDeviceConfiguration()); - break; + if (deviceProfile.getType() == DeviceProfileType.DEFAULT) { + deviceData.setConfiguration(new DefaultDeviceConfiguration()); } } if (deviceData.getTransportConfiguration() == null || !deviceProfile.getTransportType().equals(deviceData.getTransportConfiguration().getType())) { @@ -293,24 +289,20 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe return deviceData; } + @Transactional @Override public Device assignDeviceToCustomer(TenantId tenantId, DeviceId deviceId, CustomerId customerId) { Device device = findDeviceById(tenantId, deviceId); device.setCustomerId(customerId); - Device savedDevice = saveDevice(device); - cacheManager.removeDeviceFromCacheByName(tenantId, device.getName()); - cacheManager.removeDeviceFromCacheById(tenantId, device.getId()); - return savedDevice; + return saveDevice(device); } + @Transactional @Override public Device unassignDeviceFromCustomer(TenantId tenantId, DeviceId deviceId) { Device device = findDeviceById(tenantId, deviceId); device.setCustomerId(null); - Device savedDevice = saveDevice(device); - cacheManager.removeDeviceFromCacheByName(tenantId, device.getName()); - cacheManager.removeDeviceFromCacheById(tenantId, device.getId()); - return savedDevice; + return saveDevice(device); } @Transactional @@ -320,7 +312,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe validateId(deviceId, INCORRECT_DEVICE_ID + deviceId); Device device = deviceDao.findById(tenantId, deviceId.getId()); - final String deviceName = device.getName(); + DeviceCacheEvictEvent deviceCacheEvictEvent = new DeviceCacheEvictEvent(device.getTenantId(), device.getId(), device.getName(), null); try { List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), deviceId).get(); if (entityViews != null && !entityViews.isEmpty()) { @@ -339,12 +331,10 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe deviceDao.removeById(tenantId, deviceId.getId()); - cacheManager.removeDeviceFromCacheByName(tenantId, deviceName); - cacheManager.removeDeviceFromCacheById(tenantId, deviceId); + publishEvictEvent(deviceCacheEvictEvent); } - @Override public PageData findDevicesByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findDevicesByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); @@ -417,7 +407,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe return deviceDao.findDevicesByTenantIdAndIdsAsync(tenantId.getId(), toUUIDs(deviceIds)); } - + @Transactional @Override public void deleteDevicesByTenantId(TenantId tenantId) { log.trace("Executing deleteDevicesByTenantId, tenantId [{}]", tenantId); @@ -506,7 +496,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe return Futures.successfulAsList(futures); }, MoreExecutors.directExecutor()); - devices = Futures.transform(devices, new Function, List>() { + devices = Futures.transform(devices, new Function<>() { @Nullable @Override public List apply(@Nullable List deviceList) { @@ -533,7 +523,6 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe @Override public Device assignDeviceToTenant(TenantId tenantId, Device device) { log.trace("Executing assignDeviceToTenant [{}][{}]", tenantId, device); - try { List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), device.getId()).get(); if (!CollectionUtils.isEmpty(entityViews)) { @@ -554,16 +543,18 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe device.setCustomerId(null); Device savedDevice = doSaveDevice(device, null, true); + DeviceCacheEvictEvent oldTenantEvent = new DeviceCacheEvictEvent(oldTenantId, device.getId(), device.getName(), null); + DeviceCacheEvictEvent newTenantEvent = new DeviceCacheEvictEvent(savedDevice.getTenantId(), device.getId(), device.getName(), null); + // explicitly remove device with previous tenant id from cache // result device object will have different tenant id and will not remove entity from cache - cacheManager.removeDeviceFromCacheByName(oldTenantId, device.getName()); - cacheManager.removeDeviceFromCacheById(oldTenantId, device.getId()); + publishEvictEvent(oldTenantEvent); + publishEvictEvent(newTenantEvent); return savedDevice; } @Override - @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#profile.tenantId, #provisionRequest.deviceName}") @Transactional public Device saveDevice(ProvisionRequest provisionRequest, DeviceProfile profile) { Device device = new Device(); @@ -605,7 +596,8 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe throw new ProvisionFailedException(ProvisionResponseStatus.FAILURE.name()); } } - cacheManager.removeDeviceFromCacheById(savedDevice.getTenantId(), savedDevice.getId()); // eviction by name is described as annotation @CacheEvict above + + publishEvictEvent(new DeviceCacheEvictEvent(savedDevice.getTenantId(), savedDevice.getId(), provisionRequest.getDeviceName(), null)); return savedDevice; } @@ -677,7 +669,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe } private PaginatedRemover tenantDevicesRemover = - new PaginatedRemover() { + new PaginatedRemover<>() { @Override protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCacheEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCacheEvictEvent.java new file mode 100644 index 0000000000..44f4cdb25d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCacheEvictEvent.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@RequiredArgsConstructor +class EdgeCacheEvictEvent { + + private final TenantId tenantId; + private final String newName; + private final String oldName; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCacheKey.java new file mode 100644 index 0000000000..e348e4a854 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCacheKey.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; + +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +@Builder +public class EdgeCacheKey implements Serializable { + + private final TenantId tenantId; + private final String name; + + @Override + public String toString() { + return tenantId + "_" + name; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCaffeineCache.java new file mode 100644 index 0000000000..1a7956e3a0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("EdgeCache") +public class EdgeCaffeineCache extends CaffeineTbTransactionalCache { + + public EdgeCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.EDGE_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeRedisCache.java new file mode 100644 index 0000000000..ab0b44a525 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.edge; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("EdgeCache") +public class EdgeRedisCache extends RedisTbTransactionalCache { + + public EdgeRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.EDGE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 80d3a1bb03..271c74a3e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -25,11 +25,13 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.annotation.CacheEvict; -import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -47,8 +49,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainConnectionInfo; -import org.thingsboard.server.dao.cache.EntitiesCacheManager; -import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; @@ -65,7 +66,6 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.CacheConstants.EDGE_CACHE; import static org.thingsboard.server.dao.DaoUtil.toUUIDs; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validateIds; @@ -74,7 +74,7 @@ import static org.thingsboard.server.dao.service.Validator.validateString; @Service @Slf4j -public class EdgeServiceImpl extends AbstractEntityService implements EdgeService { +public class EdgeServiceImpl extends AbstractCachedEntityService implements EdgeService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_CUSTOMER_ID = "Incorrect customerId "; @@ -90,9 +90,6 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic @Autowired private UserService userService; - @Autowired - private EntitiesCacheManager cacheManager; - @Autowired private RuleChainService ruleChainService; @@ -102,6 +99,17 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic @Autowired private DataValidator edgeValidator; + @TransactionalEventListener(classes = EdgeCacheEvictEvent.class) + @Override + public void handleEvictEvent(EdgeCacheEvictEvent event) { + List keys = new ArrayList<>(2); + keys.add(new EdgeCacheKey(event.getTenantId(), event.getNewName())); + if (StringUtils.isNotEmpty(event.getOldName()) && !event.getOldName().equals(event.getNewName())) { + keys.add(new EdgeCacheKey(event.getTenantId(), event.getOldName())); + } + cache.evict(keys); + } + @Override public Edge findEdgeById(TenantId tenantId, EdgeId edgeId) { log.trace("Executing findEdgeById [{}]", edgeId); @@ -123,13 +131,13 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic return edgeDao.findByIdAsync(tenantId, edgeId.getId()); } - @Cacheable(cacheNames = EDGE_CACHE, key = "{#tenantId, #name}") @Override public Edge findEdgeByTenantIdAndName(TenantId tenantId, String name) { log.trace("Executing findEdgeByTenantIdAndName [{}][{}]", tenantId, name); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - Optional edgeOpt = edgeDao.findEdgeByTenantIdAndName(tenantId.getId(), name); - return edgeOpt.orElse(null); + return cache.getAndPutInTransaction(new EdgeCacheKey(tenantId, name), + () -> edgeDao.findEdgeByTenantIdAndName(tenantId.getId(), name) + .orElse(null), true); } @Override @@ -139,14 +147,17 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic return edgeDao.findByRoutingKey(tenantId.getId(), routingKey); } - @CacheEvict(cacheNames = EDGE_CACHE, key = "{#edge.tenantId, #edge.name}") @Override public Edge saveEdge(Edge edge) { log.trace("Executing saveEdge [{}]", edge); - edgeValidator.validate(edge, Edge::getTenantId); + Edge oldEdge = edgeValidator.validate(edge, Edge::getTenantId); + EdgeCacheEvictEvent evictEvent = new EdgeCacheEvictEvent(edge.getTenantId(), edge.getName(), oldEdge != null ? oldEdge.getName() : null); try { - return edgeDao.save(edge.getTenantId(), edge); + var savedEdge = edgeDao.save(edge.getTenantId(), edge); + publishEvictEvent(evictEvent); + return savedEdge; } catch (Exception t) { + handleEvictEvent(evictEvent); ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("edge_name_unq_key")) { @@ -182,9 +193,9 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic deleteEntityRelations(tenantId, edgeId); - cacheManager.removeEdgeFromCacheByName(edge.getTenantId(), edge.getName()); - edgeDao.removeById(tenantId, edgeId.getId()); + + publishEvictEvent(new EdgeCacheEvictEvent(edge.getTenantId(), edge.getName(), null)); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractCachedEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractCachedEntityService.java new file mode 100644 index 0000000000..9d97aab675 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractCachedEntityService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entity; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.thingsboard.server.cache.TbTransactionalCache; + +import java.io.Serializable; + +public abstract class AbstractCachedEntityService extends AbstractEntityService { + + @Autowired + protected TbTransactionalCache cache; + + @Autowired + private ApplicationEventPublisher eventPublisher; + + protected void publishEvictEvent(E event) { + if (TransactionSynchronizationManager.isActualTransactionActive()) { + eventPublisher.publishEvent(event); + } else { + handleEvictEvent(event); + } + } + + public abstract void handleEvictEvent(E event); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 2eaf68e4ca..9321fd4e9e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -40,6 +40,7 @@ public abstract class AbstractEntityService { public static final String INCORRECT_EDGE_ID = "Incorrect edgeId "; public static final String INCORRECT_PAGE_LINK = "Incorrect page link "; + @Lazy @Autowired protected RelationService relationService; @@ -47,9 +48,11 @@ public abstract class AbstractEntityService { @Autowired protected AlarmService alarmService; + @Lazy @Autowired protected EntityViewService entityViewService; + @Lazy @Autowired(required = false) protected EdgeService edgeService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCacheKey.java new file mode 100644 index 0000000000..20e9de06f6 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCacheKey.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entityview; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; + +@Getter +@EqualsAndHashCode +@Builder +public class EntityViewCacheKey implements Serializable { + + private final TenantId tenantId; + private final String name; + private final EntityId entityId; + private final EntityViewId entityViewId; + + private EntityViewCacheKey(TenantId tenantId, String name, EntityId entityId, EntityViewId entityViewId) { + this.tenantId = tenantId; + this.name = name; + this.entityId = entityId; + this.entityViewId = entityViewId; + } + + public static EntityViewCacheKey byName(TenantId tenantId, String name) { + return new EntityViewCacheKey(tenantId, name, null, null); + } + + public static EntityViewCacheKey byEntityId(TenantId tenantId, EntityId entityId) { + return new EntityViewCacheKey(tenantId, null, entityId, null); + } + + public static EntityViewCacheKey byId(EntityViewId id) { + return new EntityViewCacheKey(null, null, null, id); + } + + @Override + public String toString() { + if (entityViewId != null) { + return entityViewId.toString(); + } else if (entityId != null) { + return tenantId + "_" + entityId; + } else { + return tenantId + "_n_" + name; + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCacheValue.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCacheValue.java new file mode 100644 index 0000000000..2d086cf1d6 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCacheValue.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entityview; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.EntityView; + +import java.io.Serializable; +import java.util.List; + +@Getter +@EqualsAndHashCode +@Builder +public class EntityViewCacheValue implements Serializable { + + private static final long serialVersionUID = 1959004642076413174L; + + private final EntityView entityView; + private final List entityViews; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCaffeineCache.java new file mode 100644 index 0000000000..38d144814a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewCaffeineCache.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entityview; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("EntityViewCache") +public class EntityViewCaffeineCache extends CaffeineTbTransactionalCache { + + public EntityViewCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.ENTITY_VIEW_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java index e6020824cb..f44bafe937 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java @@ -144,7 +144,7 @@ public interface EntityViewDao extends Dao { */ PageData findEntityViewInfosByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink); - ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(UUID tenantId, UUID entityId); + List findEntityViewsByTenantIdAndEntityId(UUID tenantId, UUID entityId); /** * Find tenants entity view types. diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewEvictEvent.java new file mode 100644 index 0000000000..012377b67d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewEvictEvent.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entityview; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@RequiredArgsConstructor +class EntityViewEvictEvent { + + private final TenantId tenantId; + private final EntityViewId id; + private final EntityId newEntityId; + private final EntityId oldEntityId; + private final String newName; + private final String oldName; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewRedisCache.java new file mode 100644 index 0000000000..e2ade492ff --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewRedisCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entityview; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("EntityViewCache") +public class EntityViewRedisCache extends RedisTbTransactionalCache { + + public EntityViewRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.ENTITY_VIEW_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 5705c463cb..04bdb6697a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -16,20 +16,20 @@ package org.thingsboard.server.dao.entityview; import com.google.common.base.Function; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.entityview.EntityViewSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; @@ -42,22 +42,20 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; +import org.thingsboard.server.dao.sql.JpaExecutorService; import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.CacheConstants.ENTITY_VIEW_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; import static org.thingsboard.server.dao.service.Validator.validateString; @@ -67,7 +65,7 @@ import static org.thingsboard.server.dao.service.Validator.validateString; */ @Service @Slf4j -public class EntityViewServiceImpl extends AbstractEntityService implements EntityViewService { +public class EntityViewServiceImpl extends AbstractCachedEntityService implements EntityViewService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_PAGE_LINK = "Incorrect page link "; @@ -79,28 +77,39 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti private EntityViewDao entityViewDao; @Autowired - private CacheManager cacheManager; + private DataValidator entityViewValidator; @Autowired - private DataValidator entityViewValidator; + protected JpaExecutorService service; + @TransactionalEventListener(classes = EntityViewEvictEvent.class) @Override - public EntityView saveEntityView(EntityView entityView) { - log.trace("Executing save entity view [{}]", entityView); - entityViewValidator.validate(entityView, EntityView::getTenantId); - - if (entityView.getId() != null) { - EntityView oldEntityView = entityViewDao.findById(entityView.getTenantId(), entityView.getUuidId()); - evictCache(oldEntityView); + public void handleEvictEvent(EntityViewEvictEvent event) { + List keys = new ArrayList<>(5); + keys.add(EntityViewCacheKey.byName(event.getTenantId(), event.getNewName())); + keys.add(EntityViewCacheKey.byId(event.getId())); + keys.add(EntityViewCacheKey.byEntityId(event.getTenantId(), event.getNewEntityId())); + if (event.getOldEntityId() != null && !event.getOldEntityId().equals(event.getNewEntityId())) { + keys.add(EntityViewCacheKey.byEntityId(event.getTenantId(), event.getOldEntityId())); + } + if (StringUtils.isNotEmpty(event.getOldName()) && !event.getOldName().equals(event.getNewName())) { + keys.add(EntityViewCacheKey.byName(event.getTenantId(), event.getOldName())); } + cache.evict(keys); + } - return entityViewDao.save(entityView.getTenantId(), entityView); + @Override + public EntityView saveEntityView(EntityView entityView) { + log.trace("Executing save entity view [{}]", entityView); + EntityView old = entityViewValidator.validate(entityView, EntityView::getTenantId); + EntityView saved = entityViewDao.save(entityView.getTenantId(), entityView); + publishEvictEvent(new EntityViewEvictEvent(saved.getTenantId(), saved.getId(), saved.getEntityId(), old != null ? old.getEntityId() : null, saved.getName(), old != null ? old.getName() : null)); + return saved; } @Override public EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, CustomerId customerId) { EntityView entityView = findEntityViewById(tenantId, entityViewId); - evictCache(entityView); entityView.setCustomerId(customerId); return saveEntityView(entityView); } @@ -108,7 +117,6 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti @Override public EntityView unassignEntityViewFromCustomer(TenantId tenantId, EntityViewId entityViewId) { EntityView entityView = findEntityViewById(tenantId, entityViewId); - evictCache(entityView); entityView.setCustomerId(null); return saveEntityView(entityView); } @@ -128,21 +136,23 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti return entityViewDao.findEntityViewInfoById(tenantId, entityViewId.getId()); } - @Cacheable(cacheNames = ENTITY_VIEW_CACHE, key = "{#entityViewId}") @Override public EntityView findEntityViewById(TenantId tenantId, EntityViewId entityViewId) { log.trace("Executing findEntityViewById [{}]", entityViewId); validateId(entityViewId, INCORRECT_ENTITY_VIEW_ID + entityViewId); - return entityViewDao.findById(tenantId, entityViewId.getId()); + return cache.getAndPutInTransaction(EntityViewCacheKey.byId(entityViewId), + () -> entityViewDao.findById(tenantId, entityViewId.getId()) + , EntityViewCacheValue::getEntityView, v -> new EntityViewCacheValue(v, null), true); } - @Cacheable(cacheNames = ENTITY_VIEW_CACHE, key = "{#tenantId, #name}") @Override public EntityView findEntityViewByTenantIdAndName(TenantId tenantId, String name) { log.trace("Executing findEntityViewByTenantIdAndName [{}][{}]", tenantId, name); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - Optional entityViewOpt = entityViewDao.findEntityViewByTenantIdAndName(tenantId.getId(), name); - return entityViewOpt.orElse(null); + return cache.getAndPutInTransaction(EntityViewCacheKey.byName(tenantId, name), + () -> entityViewDao.findEntityViewByTenantIdAndName(tenantId.getId(), name).orElse(null) + , EntityViewCacheValue::getEntityView, v -> new EntityViewCacheValue(v, null), true); + } @Override @@ -265,31 +275,9 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti validateId(tenantId, INCORRECT_TENANT_ID + tenantId); validateId(entityId.getId(), "Incorrect entityId" + entityId); - List tenantIdAndEntityId = new ArrayList<>(); - tenantIdAndEntityId.add(tenantId); - tenantIdAndEntityId.add(entityId); - - Cache cache = cacheManager.getCache(ENTITY_VIEW_CACHE); - @SuppressWarnings("unchecked") - List fromCache = cache.get(tenantIdAndEntityId, List.class); - if (fromCache != null) { - return Futures.immediateFuture(fromCache); - } else { - ListenableFuture> entityViewsFuture = entityViewDao.findEntityViewsByTenantIdAndEntityIdAsync(tenantId.getId(), entityId.getId()); - Futures.addCallback(entityViewsFuture, - new FutureCallback>() { - @Override - public void onSuccess(@Nullable List result) { - cache.putIfAbsent(tenantIdAndEntityId, result); - } - - @Override - public void onFailure(Throwable t) { - log.error("Error while finding entity views by tenantId and entityId", t); - } - }, MoreExecutors.directExecutor()); - return entityViewsFuture; - } + return service.submit(() -> cache.getAndPutInTransaction(EntityViewCacheKey.byEntityId(tenantId, entityId), + () -> entityViewDao.findEntityViewsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()), + EntityViewCacheValue::getEntityViews, v -> new EntityViewCacheValue(null, v), true)); } @Override @@ -298,8 +286,8 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti validateId(entityViewId, INCORRECT_ENTITY_VIEW_ID + entityViewId); deleteEntityRelations(tenantId, entityViewId); EntityView entityView = entityViewDao.findById(tenantId, entityViewId.getId()); - evictCache(entityView); entityViewDao.removeById(tenantId, entityViewId.getId()); + publishEvictEvent(new EntityViewEvictEvent(entityView.getTenantId(), entityView.getId(), entityView.getEntityId(), null, entityView.getName(), null)); } @Override @@ -410,11 +398,4 @@ public class EntityViewServiceImpl extends AbstractEntityService implements Enti unassignEntityViewFromCustomer(tenantId, new EntityViewId(entity.getUuidId())); } }; - - private void evictCache(EntityView entityView) { - Cache cache = cacheManager.getCache(ENTITY_VIEW_CACHE); - cache.evict(Arrays.asList(entityView.getTenantId(), entityView.getEntityId())); - cache.evict(Arrays.asList(entityView.getTenantId(), entityView.getName())); - cache.evict(Arrays.asList(entityView.getId())); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index f3574fb419..86eb0d91ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -19,13 +19,11 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Event; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.Dao; import java.util.List; -import java.util.Optional; import java.util.UUID; /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEdgeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEdgeEntity.java index 40ef7d82ab..6709fef21f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEdgeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEdgeEntity.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.model.sql; -import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java index 42b3f1d11a..a8a31cd892 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java @@ -25,7 +25,6 @@ import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTenantEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTenantEntity.java index c3a0e8396a..2ddb065ea3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTenantEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractTenantEntity.java @@ -21,7 +21,6 @@ import lombok.EqualsAndHashCode; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.dao.model.BaseSqlEntity; @@ -30,9 +29,7 @@ import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; -import javax.persistence.Entity; import javax.persistence.MappedSuperclass; -import javax.persistence.Table; import java.util.UUID; @Data diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java index 64956bf871..42b2777ca4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java @@ -20,9 +20,9 @@ import lombok.EqualsAndHashCode; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OAuth2ClientRegistrationTemplateEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OAuth2ClientRegistrationTemplateEntity.java index 0a7e1fac33..40344025ba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OAuth2ClientRegistrationTemplateEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OAuth2ClientRegistrationTemplateEntity.java @@ -21,16 +21,21 @@ import lombok.EqualsAndHashCode; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.id.OAuth2ClientRegistrationTemplateId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.oauth2.*; +import org.thingsboard.server.common.data.oauth2.MapperType; +import org.thingsboard.server.common.data.oauth2.OAuth2BasicMapperConfig; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; +import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; +import org.thingsboard.server.common.data.oauth2.TenantNameStrategyType; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.mapping.JsonStringType; -import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; -import javax.persistence.*; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Table; import java.util.Arrays; -import java.util.UUID; @Data @EqualsAndHashCode(callSuper = true) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java index 9d61cf52f1..99abac10fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java @@ -21,11 +21,11 @@ import lombok.EqualsAndHashCode; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageInfoEntity.java index d5f34c922e..286c7f6605 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageInfoEntity.java @@ -22,12 +22,11 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.OtaPackageInfo; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeStateEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeStateEntity.java index bb62ecdd4e..0ba56a73ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeStateEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeStateEntity.java @@ -15,10 +15,8 @@ */ package org.thingsboard.server.dao.model.sql; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import lombok.EqualsAndHashCode; -import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleNodeId; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java index 8613c7ced5..6f7b094464 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TenantProfileEntity.java @@ -21,13 +21,13 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.util.mapping.JsonBinaryType; import javax.persistence.Column; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java index 956b8d7cd8..d03d8d8541 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetTypeEntity.java @@ -28,7 +28,6 @@ import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; -import java.util.UUID; @Data @EqualsAndHashCode(callSuper = true) diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java index 605a11d6ce..f557e9c253 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.oauth2; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Utils.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Utils.java index 1f3f24856c..a00f94d7e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Utils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Utils.java @@ -18,9 +18,19 @@ package org.thingsboard.server.dao.oauth2; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.id.OAuth2ParamsId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.oauth2.*; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2Domain; +import org.thingsboard.server.common.data.oauth2.OAuth2DomainInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2Info; +import org.thingsboard.server.common.data.oauth2.OAuth2Mobile; +import org.thingsboard.server.common.data.oauth2.OAuth2MobileInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2Params; +import org.thingsboard.server.common.data.oauth2.OAuth2ParamsInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2Registration; +import org.thingsboard.server.common.data.oauth2.OAuth2RegistrationInfo; -import java.util.*; +import java.util.Comparator; +import java.util.List; import java.util.stream.Collectors; public class OAuth2Utils { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index ecf8d68267..4d0bb18a43 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -21,10 +21,10 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -45,40 +46,47 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service @Slf4j @RequiredArgsConstructor -public class BaseOtaPackageService implements OtaPackageService { +public class BaseOtaPackageService extends AbstractCachedEntityService implements OtaPackageService { public static final String INCORRECT_OTA_PACKAGE_ID = "Incorrect otaPackageId "; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; private final OtaPackageDao otaPackageDao; private final OtaPackageInfoDao otaPackageInfoDao; - private final CacheManager cacheManager; private final OtaPackageDataCache otaPackageDataCache; private final DataValidator otaPackageInfoValidator; private final DataValidator otaPackageValidator; + @TransactionalEventListener(classes = OtaPackageCacheEvictEvent.class) + @Override + public void handleEvictEvent(OtaPackageCacheEvictEvent event) { + cache.evict(new OtaPackageCacheKey(event.getId())); + otaPackageDataCache.evict(event.getId().toString()); + } + @Override public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl) { log.trace("Executing saveOtaPackageInfo [{}]", otaPackageInfo); - if(isUrl && (StringUtils.isEmpty(otaPackageInfo.getUrl()) || otaPackageInfo.getUrl().trim().length() == 0)) { + if (isUrl && (StringUtils.isEmpty(otaPackageInfo.getUrl()) || otaPackageInfo.getUrl().trim().length() == 0)) { throw new DataValidationException("Ota package URL should be specified!"); } otaPackageInfoValidator.validate(otaPackageInfo, OtaPackageInfo::getTenantId); + OtaPackageId otaPackageId = otaPackageInfo.getId(); try { - OtaPackageId otaPackageId = otaPackageInfo.getId(); + var result = otaPackageInfoDao.save(otaPackageInfo.getTenantId(), otaPackageInfo); if (otaPackageId != null) { - Cache cache = cacheManager.getCache(OTA_PACKAGE_CACHE); - cache.evict(toOtaPackageInfoKey(otaPackageId)); - otaPackageDataCache.evict(otaPackageId.toString()); + publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); } - return otaPackageInfoDao.save(otaPackageInfo.getTenantId(), otaPackageInfo); + return result; } catch (Exception t) { + if (otaPackageId != null) { + handleEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); + } ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("ota_package_tenant_title_version_unq_key")) { throw new DataValidationException("OtaPackage with such title and version already exists!"); @@ -92,15 +100,17 @@ public class BaseOtaPackageService implements OtaPackageService { public OtaPackage saveOtaPackage(OtaPackage otaPackage) { log.trace("Executing saveOtaPackage [{}]", otaPackage); otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); + OtaPackageId otaPackageId = otaPackage.getId(); try { - OtaPackageId otaPackageId = otaPackage.getId(); + var result = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); if (otaPackageId != null) { - Cache cache = cacheManager.getCache(OTA_PACKAGE_CACHE); - cache.evict(toOtaPackageInfoKey(otaPackageId)); - otaPackageDataCache.evict(otaPackageId.toString()); + publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); } - return otaPackageDao.save(otaPackage.getTenantId(), otaPackage); + return result; } catch (Exception t) { + if (otaPackageId != null) { + handleEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); + } ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("ota_package_tenant_title_version_unq_key")) { throw new DataValidationException("OtaPackage with such title and version already exists!"); @@ -149,11 +159,11 @@ public class BaseOtaPackageService implements OtaPackageService { } @Override - @Cacheable(cacheNames = OTA_PACKAGE_CACHE, key = "{#otaPackageId}") public OtaPackageInfo findOtaPackageInfoById(TenantId tenantId, OtaPackageId otaPackageId) { log.trace("Executing findOtaPackageInfoById [{}]", otaPackageId); validateId(otaPackageId, INCORRECT_OTA_PACKAGE_ID + otaPackageId); - return otaPackageInfoDao.findById(tenantId, otaPackageId.getId()); + return cache.getAndPutInTransaction(new OtaPackageCacheKey(otaPackageId), + () -> otaPackageInfoDao.findById(tenantId, otaPackageId.getId()), true); } @Override @@ -184,10 +194,8 @@ public class BaseOtaPackageService implements OtaPackageService { log.trace("Executing deleteOtaPackage [{}]", otaPackageId); validateId(otaPackageId, INCORRECT_OTA_PACKAGE_ID + otaPackageId); try { - Cache cache = cacheManager.getCache(OTA_PACKAGE_CACHE); - cache.evict(toOtaPackageInfoKey(otaPackageId)); - otaPackageDataCache.evict(otaPackageId.toString()); otaPackageDao.removeById(tenantId, otaPackageId.getId()); + publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("fk_firmware_device")) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCacheEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCacheEvictEvent.java new file mode 100644 index 0000000000..c4bbe77a52 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCacheEvictEvent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.ota; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.OtaPackageId; + +@Data +@RequiredArgsConstructor +class OtaPackageCacheEvictEvent { + + private final OtaPackageId id; + +} diff --git a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCacheKey.java similarity index 56% rename from application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java rename to dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCacheKey.java index cb99c21165..e238c70f2c 100644 --- a/application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCacheKey.java @@ -13,18 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.rules; +package org.thingsboard.server.dao.ota; -import org.junit.BeforeClass; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; -import org.thingsboard.server.queue.memory.InMemoryStorage; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.OtaPackageId; -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({ - "org.thingsboard.server.rules.flow.sql.*Test", - "org.thingsboard.server.rules.lifecycle.sql.*Test", -}) -public class RuleEngineSqlTestSuite { +import java.io.Serializable; + +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +@Builder +public class OtaPackageCacheKey implements Serializable { + + private final OtaPackageId id; + + @Override + public String toString() { + return id.toString(); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCaffeineCache.java new file mode 100644 index 0000000000..b1b5faa50d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.ota; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("OtaPackageCache") +public class OtaPackageCaffeineCache extends CaffeineTbTransactionalCache { + + public OtaPackageCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.OTA_PACKAGE_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index 445210ca3b..a807f501bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.ota; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; -import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java index 5d2c51ed07..9099b57511 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java @@ -16,10 +16,10 @@ package org.thingsboard.server.dao.ota; import org.thingsboard.server.common.data.OtaPackageInfo; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageRedisCache.java new file mode 100644 index 0000000000..1257932b79 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.ota; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("OtaPackageCache") +public class OtaPackageRedisCache extends RedisTbTransactionalCache { + + public OtaPackageRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.OTA_PACKAGE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index da119d453e..912a3393c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -16,25 +16,22 @@ package org.thingsboard.server.dao.relation; import com.google.common.base.Function; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.CacheEvict; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.cache.annotation.Caching; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Lazy; import org.springframework.dao.ConcurrencyFailureException; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.EntityType; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationInfo; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; @@ -46,18 +43,16 @@ import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.ConstraintValidator; +import org.thingsboard.server.dao.sql.JpaExecutorService; -import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; import java.util.function.BiConsumer; -import static org.thingsboard.server.common.data.CacheConstants.RELATIONS_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; /** @@ -67,14 +62,32 @@ import static org.thingsboard.server.dao.service.Validator.validateId; @Slf4j public class BaseRelationService implements RelationService { - @Autowired - private RelationDao relationDao; - - @Autowired - private EntityService entityService; - - @Autowired - private CacheManager cacheManager; + private final RelationDao relationDao; + private final EntityService entityService; + private final TbTransactionalCache cache; + private final ApplicationEventPublisher eventPublisher; + private final JpaExecutorService executor; + + public BaseRelationService(RelationDao relationDao, @Lazy EntityService entityService, + TbTransactionalCache cache, + ApplicationEventPublisher eventPublisher, JpaExecutorService executor) { + this.relationDao = relationDao; + this.entityService = entityService; + this.cache = cache; + this.eventPublisher = eventPublisher; + this.executor = executor; + } + + @TransactionalEventListener(classes = EntityRelationEvent.class) + public void handleEvictEvent(EntityRelationEvent event) { + List keys = new ArrayList<>(5); + keys.add(new RelationCacheKey(event.getFrom(), event.getTo(), event.getType(), event.getTypeGroup())); + keys.add(new RelationCacheKey(event.getFrom(), null, event.getType(), event.getTypeGroup(), EntitySearchDirection.FROM)); + keys.add(new RelationCacheKey(event.getFrom(), null, null, event.getTypeGroup(), EntitySearchDirection.FROM)); + keys.add(new RelationCacheKey(null, event.getTo(), event.getType(), event.getTypeGroup(), EntitySearchDirection.TO)); + keys.add(new RelationCacheKey(null, event.getTo(), null, event.getTypeGroup(), EntitySearchDirection.TO)); + cache.evict(keys); + } @Override public ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { @@ -83,112 +96,81 @@ public class BaseRelationService implements RelationService { return relationDao.checkRelation(tenantId, from, to, relationType, typeGroup); } - @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#from, #to, #relationType, #typeGroup}") @Override public EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { - try { - return getRelationAsync(tenantId, from, to, relationType, typeGroup).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - } - - @Override - public ListenableFuture getRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { log.trace("Executing EntityRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); - return relationDao.getRelation(tenantId, from, to, relationType, typeGroup); + RelationCacheKey cacheKey = new RelationCacheKey(from, to, relationType, typeGroup); + return cache.getAndPutInTransaction(cacheKey, + () -> { + log.trace("FETCH EntityRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); + return relationDao.getRelation(tenantId, from, to, relationType, typeGroup); + }, + RelationCacheValue::getRelation, + relations -> RelationCacheValue.builder().relation(relations).build(), false); } - @Caching(evict = { - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.to, #relation.type, #relation.typeGroup}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.type, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.typeGroup, 'TO'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.type, #relation.typeGroup, 'TO'}") - }) @Override public boolean saveRelation(TenantId tenantId, EntityRelation relation) { log.trace("Executing saveRelation [{}]", relation); validate(relation); - return relationDao.saveRelation(tenantId, relation); + var result = relationDao.saveRelation(tenantId, relation); + publishEvictEvent(EntityRelationEvent.from(relation)); + return result; } - @Caching(evict = { - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.to, #relation.type, #relation.typeGroup}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.type, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.typeGroup, 'TO'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.type, #relation.typeGroup, 'TO'}") - }) @Override public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { log.trace("Executing saveRelationAsync [{}]", relation); validate(relation); - return relationDao.saveRelationAsync(tenantId, relation); + var future = relationDao.saveRelationAsync(tenantId, relation); + future.addListener(() -> handleEvictEvent(EntityRelationEvent.from(relation)), MoreExecutors.directExecutor()); + return future; } - @Caching(evict = { - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.to, #relation.type, #relation.typeGroup}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.type, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.typeGroup, 'TO'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.type, #relation.typeGroup, 'TO'}") - }) @Override public boolean deleteRelation(TenantId tenantId, EntityRelation relation) { - log.trace("Executing deleteRelation [{}]", relation); + log.trace("Executing DeleteRelation [{}]", relation); validate(relation); - return relationDao.deleteRelation(tenantId, relation); + var result = relationDao.deleteRelation(tenantId, relation); + //TODO: evict cache only if the relation was deleted. Note: relationDao.deleteRelation requires improvement. + publishEvictEvent(EntityRelationEvent.from(relation)); + return result; } - @Caching(evict = { - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.to, #relation.type, #relation.typeGroup}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.type, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.from, #relation.typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.typeGroup, 'TO'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#relation.to, #relation.type, #relation.typeGroup, 'TO'}") - }) @Override public ListenableFuture deleteRelationAsync(TenantId tenantId, EntityRelation relation) { log.trace("Executing deleteRelationAsync [{}]", relation); validate(relation); - return relationDao.deleteRelationAsync(tenantId, relation); + var future = relationDao.deleteRelationAsync(tenantId, relation); + future.addListener(() -> handleEvictEvent(EntityRelationEvent.from(relation)), MoreExecutors.directExecutor()); + return future; } - @Caching(evict = { - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#from, #to, #relationType, #typeGroup}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#from, #relationType, #typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#from, #typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#to, #typeGroup, 'TO'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#to, #relationType, #typeGroup, 'TO'}") - }) @Override public boolean deleteRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { log.trace("Executing deleteRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); - return relationDao.deleteRelation(tenantId, from, to, relationType, typeGroup); + var result = relationDao.deleteRelation(tenantId, from, to, relationType, typeGroup); + //TODO: evict cache only if the relation was deleted. Note: relationDao.deleteRelation requires improvement. + publishEvictEvent(new EntityRelationEvent(from, to, relationType, typeGroup)); + return result; } - @Caching(evict = { - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#from, #to, #relationType, #typeGroup}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#from, #relationType, #typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#from, #typeGroup, 'FROM'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#to, #typeGroup, 'TO'}"), - @CacheEvict(cacheNames = RELATIONS_CACHE, key = "{#to, #relationType, #typeGroup, 'TO'}") - }) @Override public ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { log.trace("Executing deleteRelationAsync [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); - return relationDao.deleteRelationAsync(tenantId, from, to, relationType, typeGroup); + var future = relationDao.deleteRelationAsync(tenantId, from, to, relationType, typeGroup); + EntityRelationEvent event = new EntityRelationEvent(from, to, relationType, typeGroup); + future.addListener(() -> handleEvictEvent(event), MoreExecutors.directExecutor()); + return future; } @Override public void deleteEntityRelations(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteEntityRelations [{}]", entityId); validate(entityId); - final Cache cache = cacheManager.getCache(RELATIONS_CACHE); List inboundRelations = new ArrayList<>(); for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { inboundRelations.addAll(relationDao.findAllByTo(tenantId, entityId, typeGroup)); @@ -200,11 +182,11 @@ public class BaseRelationService implements RelationService { } for (EntityRelation relation : inboundRelations) { - delete(tenantId, cache, relation, true); + delete(tenantId, relation, true); } for (EntityRelation relation : outboundRelations) { - delete(tenantId, cache, relation, false); + delete(tenantId, relation, false); } relationDao.deleteOutboundRelations(tenantId, entityId); @@ -213,32 +195,31 @@ public class BaseRelationService implements RelationService { @Override public ListenableFuture deleteEntityRelationsAsync(TenantId tenantId, EntityId entityId) { - Cache cache = cacheManager.getCache(RELATIONS_CACHE); log.trace("Executing deleteEntityRelationsAsync [{}]", entityId); validate(entityId); List>> inboundRelationsList = new ArrayList<>(); for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelationsList.add(relationDao.findAllByToAsync(tenantId, entityId, typeGroup)); + inboundRelationsList.add(executor.submit(() -> relationDao.findAllByTo(tenantId, entityId, typeGroup))); } ListenableFuture>> inboundRelations = Futures.allAsList(inboundRelationsList); List>> outboundRelationsList = new ArrayList<>(); for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelationsList.add(relationDao.findAllByFromAsync(tenantId, entityId, typeGroup)); + outboundRelationsList.add(executor.submit(() -> relationDao.findAllByFrom(tenantId, entityId, typeGroup))); } ListenableFuture>> outboundRelations = Futures.allAsList(outboundRelationsList); ListenableFuture> inboundDeletions = Futures.transformAsync(inboundRelations, relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, cache, true); + List> results = deleteRelationGroupsAsync(tenantId, relations, true); return Futures.allAsList(results); }, MoreExecutors.directExecutor()); ListenableFuture> outboundDeletions = Futures.transformAsync(outboundRelations, relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, cache, false); + List> results = deleteRelationGroupsAsync(tenantId, relations, false); return Futures.allAsList(results); }, MoreExecutors.directExecutor()); @@ -250,25 +231,29 @@ public class BaseRelationService implements RelationService { result -> null, MoreExecutors.directExecutor()); } - private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, Cache cache, boolean deleteFromDb) { + private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, boolean deleteFromDb) { List> results = new ArrayList<>(); for (List relationList : relations) { - relationList.forEach(relation -> results.add(deleteAsync(tenantId, cache, relation, deleteFromDb))); + relationList.forEach(relation -> results.add(deleteAsync(tenantId, relation, deleteFromDb))); } return results; } - private ListenableFuture deleteAsync(TenantId tenantId, Cache cache, EntityRelation relation, boolean deleteFromDb) { - cacheEviction(relation, cache); + private ListenableFuture deleteAsync(TenantId tenantId, EntityRelation relation, boolean deleteFromDb) { if (deleteFromDb) { - return relationDao.deleteRelationAsync(tenantId, relation); + return Futures.transform(relationDao.deleteRelationAsync(tenantId, relation), + bool -> { + handleEvictEvent(EntityRelationEvent.from(relation)); + return bool; + }, MoreExecutors.directExecutor()); } else { + handleEvictEvent(EntityRelationEvent.from(relation)); return Futures.immediateFuture(false); } } - boolean delete(TenantId tenantId, Cache cache, EntityRelation relation, boolean deleteFromDb) { - cacheEviction(relation, cache); + boolean delete(TenantId tenantId, EntityRelation relation, boolean deleteFromDb) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); if (deleteFromDb) { try { return relationDao.deleteRelation(tenantId, relation); @@ -279,51 +264,15 @@ public class BaseRelationService implements RelationService { return false; } - private void cacheEviction(EntityRelation relation, Cache cache) { - List fromToTypeAndTypeGroup = new ArrayList<>(); - fromToTypeAndTypeGroup.add(relation.getFrom()); - fromToTypeAndTypeGroup.add(relation.getTo()); - fromToTypeAndTypeGroup.add(relation.getType()); - fromToTypeAndTypeGroup.add(relation.getTypeGroup()); - cache.evict(fromToTypeAndTypeGroup); - - List fromTypeAndTypeGroup = new ArrayList<>(); - fromTypeAndTypeGroup.add(relation.getFrom()); - fromTypeAndTypeGroup.add(relation.getType()); - fromTypeAndTypeGroup.add(relation.getTypeGroup()); - fromTypeAndTypeGroup.add(EntitySearchDirection.FROM.name()); - cache.evict(fromTypeAndTypeGroup); - - List fromAndTypeGroup = new ArrayList<>(); - fromAndTypeGroup.add(relation.getFrom()); - fromAndTypeGroup.add(relation.getTypeGroup()); - fromAndTypeGroup.add(EntitySearchDirection.FROM.name()); - cache.evict(fromAndTypeGroup); - - List toAndTypeGroup = new ArrayList<>(); - toAndTypeGroup.add(relation.getTo()); - toAndTypeGroup.add(relation.getTypeGroup()); - toAndTypeGroup.add(EntitySearchDirection.TO.name()); - cache.evict(toAndTypeGroup); - - List toTypeAndTypeGroup = new ArrayList<>(); - toTypeAndTypeGroup.add(relation.getTo()); - toTypeAndTypeGroup.add(relation.getType()); - toTypeAndTypeGroup.add(relation.getTypeGroup()); - toTypeAndTypeGroup.add(EntitySearchDirection.TO.name()); - cache.evict(toTypeAndTypeGroup); - } - - @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#from, #typeGroup, 'FROM'}") @Override public List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup) { validate(from); validateTypeGroup(typeGroup); - try { - return relationDao.findAllByFromAsync(tenantId, from, typeGroup).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } + RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).typeGroup(typeGroup).direction(EntitySearchDirection.FROM).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findAllByFrom(tenantId, from, typeGroup), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); } @Override @@ -332,30 +281,13 @@ public class BaseRelationService implements RelationService { validate(from); validateTypeGroup(typeGroup); - List fromAndTypeGroup = new ArrayList<>(); - fromAndTypeGroup.add(from); - fromAndTypeGroup.add(typeGroup); - fromAndTypeGroup.add(EntitySearchDirection.FROM.name()); + var cacheValue = cache.get(RelationCacheKey.builder().from(from).typeGroup(typeGroup).direction(EntitySearchDirection.FROM).build()); - Cache cache = cacheManager.getCache(RELATIONS_CACHE); - @SuppressWarnings("unchecked") - List fromCache = cache.get(fromAndTypeGroup, List.class); - if (fromCache != null) { - return Futures.immediateFuture(fromCache); + if (cacheValue != null && cacheValue.get() != null) { + return Futures.immediateFuture(cacheValue.get().getRelations()); } else { - ListenableFuture> relationsFuture = relationDao.findAllByFromAsync(tenantId, from, typeGroup); - Futures.addCallback(relationsFuture, - new FutureCallback>() { - @Override - public void onSuccess(@Nullable List result) { - cache.putIfAbsent(fromAndTypeGroup, result); - } - - @Override - public void onFailure(Throwable t) { - } - }, MoreExecutors.directExecutor()); - return relationsFuture; + //Disabled cache put for the async requests due to limitations of the cache implementation (Redis lib does not support thread-safe transactions) + return executor.submit(() -> findByFrom(tenantId, from, typeGroup)); } } @@ -364,7 +296,7 @@ public class BaseRelationService implements RelationService { log.trace("Executing findInfoByFrom [{}][{}]", from, typeGroup); validate(from); validateTypeGroup(typeGroup); - ListenableFuture> relations = relationDao.findAllByFromAsync(tenantId, from, typeGroup); + ListenableFuture> relations = executor.submit(() -> relationDao.findAllByFrom(tenantId, from, typeGroup)); return Futures.transformAsync(relations, relations1 -> { List> futures = new ArrayList<>(); @@ -377,14 +309,13 @@ public class BaseRelationService implements RelationService { }, MoreExecutors.directExecutor()); } - @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#from, #relationType, #typeGroup, 'FROM'}") @Override public List findByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup) { - try { - return findByFromAndTypeAsync(tenantId, from, relationType, typeGroup).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } + RelationCacheKey cacheKey = RelationCacheKey.builder().from(from).type(relationType).typeGroup(typeGroup).direction(EntitySearchDirection.FROM).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findAllByFromAndType(tenantId, from, relationType, typeGroup), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); } @Override @@ -393,52 +324,27 @@ public class BaseRelationService implements RelationService { validate(from); validateType(relationType); validateTypeGroup(typeGroup); - return relationDao.findAllByFromAndType(tenantId, from, relationType, typeGroup); + return executor.submit(() -> findByFromAndType(tenantId, from, relationType, typeGroup)); } - @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#to, #typeGroup, 'TO'}") @Override public List findByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { validate(to); validateTypeGroup(typeGroup); - try { - return relationDao.findAllByToAsync(tenantId, to, typeGroup).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } + RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).typeGroup(typeGroup).direction(EntitySearchDirection.TO).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findAllByTo(tenantId, to, typeGroup), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); + } @Override public ListenableFuture> findByToAsync(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { - log.trace("Executing findByTo [{}][{}]", to, typeGroup); + log.trace("Executing findByToAsync [{}][{}]", to, typeGroup); validate(to); validateTypeGroup(typeGroup); - - List toAndTypeGroup = new ArrayList<>(); - toAndTypeGroup.add(to); - toAndTypeGroup.add(typeGroup); - toAndTypeGroup.add(EntitySearchDirection.TO.name()); - - Cache cache = cacheManager.getCache(RELATIONS_CACHE); - @SuppressWarnings("unchecked") - List fromCache = cache.get(toAndTypeGroup, List.class); - if (fromCache != null) { - return Futures.immediateFuture(fromCache); - } else { - ListenableFuture> relationsFuture = relationDao.findAllByToAsync(tenantId, to, typeGroup); - Futures.addCallback(relationsFuture, - new FutureCallback>() { - @Override - public void onSuccess(@Nullable List result) { - cache.putIfAbsent(toAndTypeGroup, result); - } - - @Override - public void onFailure(Throwable t) { - } - }, MoreExecutors.directExecutor()); - return relationsFuture; - } + return executor.submit(() -> findByTo(tenantId, to, typeGroup)); } @Override @@ -446,7 +352,7 @@ public class BaseRelationService implements RelationService { log.trace("Executing findInfoByTo [{}][{}]", to, typeGroup); validate(to); validateTypeGroup(typeGroup); - ListenableFuture> relations = relationDao.findAllByToAsync(tenantId, to, typeGroup); + ListenableFuture> relations = findByToAsync(tenantId, to, typeGroup); return Futures.transformAsync(relations, relations1 -> { List> futures = new ArrayList<>(); @@ -470,23 +376,27 @@ public class BaseRelationService implements RelationService { }, MoreExecutors.directExecutor()); } - @Cacheable(cacheNames = RELATIONS_CACHE, key = "{#to, #relationType, #typeGroup, 'TO'}") @Override public List findByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { - try { - return findByToAndTypeAsync(tenantId, to, relationType, typeGroup).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } + log.trace("Executing findByToAndType [{}][{}][{}]", to, relationType, typeGroup); + validate(to); + validateType(relationType); + validateTypeGroup(typeGroup); + RelationCacheKey cacheKey = RelationCacheKey.builder().to(to).type(relationType).typeGroup(typeGroup).direction(EntitySearchDirection.TO).build(); + return cache.getAndPutInTransaction(cacheKey, + () -> relationDao.findAllByToAndType(tenantId, to, relationType, typeGroup), + RelationCacheValue::getRelations, + relations -> RelationCacheValue.builder().relations(relations).build(), false); + } @Override public ListenableFuture> findByToAndTypeAsync(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { - log.trace("Executing findByToAndType [{}][{}][{}]", to, relationType, typeGroup); + log.trace("Executing findByToAndTypeAsync [{}][{}][{}]", to, relationType, typeGroup); validate(to); validateType(relationType); validateTypeGroup(typeGroup); - return relationDao.findAllByToAndType(tenantId, to, relationType, typeGroup); + return executor.submit(() -> findByToAndType(tenantId, to, relationType, typeGroup)); } @Override @@ -547,7 +457,7 @@ public class BaseRelationService implements RelationService { @Override public void removeRelations(TenantId tenantId, EntityId entityId) { - Cache cache = cacheManager.getCache(RELATIONS_CACHE); + log.trace("removeRelations {}", entityId); List relations = new ArrayList<>(); for (RelationTypeGroup relationTypeGroup : RelationTypeGroup.values()) { @@ -556,7 +466,6 @@ public class BaseRelationService implements RelationService { } for (EntityRelation relation : relations) { - cacheEviction(relation, cache); deleteRelation(tenantId, relation); } } @@ -605,21 +514,6 @@ public class BaseRelationService implements RelationService { } } - private Function, Boolean> getListToBooleanFunction() { - return new Function, Boolean>() { - @Nullable - @Override - public Boolean apply(@Nullable List results) { - for (Boolean result : results) { - if (result == null || !result) { - return false; - } - } - return true; - } - }; - } - private boolean matchFilters(List filters, EntityRelation relation, EntitySearchDirection direction) { for (RelationEntityTypeFilter filter : filters) { if (match(filter, relation, direction)) { @@ -692,4 +586,13 @@ public class BaseRelationService implements RelationService { } return relations; } + + private void publishEvictEvent(EntityRelationEvent event) { + if (TransactionSynchronizationManager.isActualTransactionActive()) { + eventPublisher.publishEvent(event); + } else { + handleEvictEvent(event); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/EntityRelationEvent.java b/dao/src/main/java/org/thingsboard/server/dao/relation/EntityRelationEvent.java new file mode 100644 index 0000000000..ff2b5c5743 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/EntityRelationEvent.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.relation; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; + +@RequiredArgsConstructor +public class EntityRelationEvent { + @Getter + private final EntityId from; + @Getter + private final EntityId to; + @Getter + private final String type; + @Getter + private final RelationTypeGroup typeGroup; + + public static EntityRelationEvent from(EntityRelation relation) { + return new EntityRelationEvent(relation.getFrom(), relation.getTo(), relation.getType(), relation.getTypeGroup()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java new file mode 100644 index 0000000000..f45c8a1344 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheKey.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.relation; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; + +import java.io.Serializable; + +@EqualsAndHashCode +@Getter +@RequiredArgsConstructor +@Builder +public class RelationCacheKey implements Serializable { + + private static final long serialVersionUID = 3911151843961657570L; + + private final EntityId from; + private final EntityId to; + private final String type; + private final RelationTypeGroup typeGroup; + private final EntitySearchDirection direction; + + public RelationCacheKey(EntityId from, EntityId to, String type, RelationTypeGroup typeGroup) { + this(from, to, type, typeGroup, null); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + boolean first = add(sb, true, from); + first = add(sb, first, to); + first = add(sb, first, type); + first = add(sb, first, typeGroup); + add(sb, first, direction); + return sb.toString(); + } + + private boolean add(StringBuilder sb, boolean first, Object param) { + if (param != null) { + if (!first) { + sb.append("_"); + } + first = false; + sb.append(param); + } + return first; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheValue.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheValue.java new file mode 100644 index 0000000000..b596033b1c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCacheValue.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.relation; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.relation.EntityRelation; + +import java.io.Serializable; +import java.util.List; + +@EqualsAndHashCode +@Getter +@RequiredArgsConstructor +@Builder +public class RelationCacheValue implements Serializable { + + private static final long serialVersionUID = 3911151843961657570L; + + private final EntityRelation relation; + private final List relations; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCaffeineCache.java new file mode 100644 index 0000000000..a2fad34a56 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationCaffeineCache.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.relation; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("RelationCache") +public class RelationCaffeineCache extends CaffeineTbTransactionalCache { + + public RelationCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.RELATIONS_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index df285602ba..e15f4f3d69 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -29,21 +29,17 @@ import java.util.List; */ public interface RelationDao { - ListenableFuture> findAllByFromAsync(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); - List findAllByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); - ListenableFuture> findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); - - ListenableFuture> findAllByToAsync(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); - ListenableFuture> findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); + List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - ListenableFuture getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean saveRelation(TenantId tenantId, EntityRelation relation); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationRedisCache.java new file mode 100644 index 0000000000..b849d36f63 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationRedisCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.relation; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("RelationCache") +public class RelationRedisCache extends RedisTbTransactionalCache { + + public RelationRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.RELATIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 542a487949..f53f8a9e9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -19,13 +19,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Collection; -import java.util.List; import java.util.UUID; /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java index f761f9f7f9..9bad52463a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeStateDao.java @@ -15,9 +15,6 @@ */ package org.thingsboard.server.dao.rule; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.RuleNodeId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNodeState; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java index e14e414ebb..1b27a2751f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java @@ -36,7 +36,8 @@ public abstract class DataValidator> { private static final Pattern EMAIL_PATTERN = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}$", Pattern.CASE_INSENSITIVE); - public void validate(D data, Function tenantIdFunction) { + // Returns old instance of the same object that is fetched during validation. + public D validate(D data, Function tenantIdFunction) { try { if (data == null) { throw new DataValidationException("Data object can't be null!"); @@ -46,11 +47,14 @@ public abstract class DataValidator> { TenantId tenantId = tenantIdFunction.apply(data); validateDataImpl(tenantId, data); + D old; if (data.getId() == null) { validateCreate(tenantId, data); + old = null; } else { - validateUpdate(tenantId, data); + old = validateUpdate(tenantId, data); } + return old; } catch (DataValidationException e) { log.error("Data object is invalid: [{}]", e.getMessage()); throw e; @@ -63,7 +67,8 @@ public abstract class DataValidator> { protected void validateCreate(TenantId tenantId, D data) { } - protected void validateUpdate(TenantId tenantId, D data) { + protected D validateUpdate(TenantId tenantId, D data) { + return null; } protected boolean isSameData(D existentData, D actualData) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AdminSettingsDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AdminSettingsDataValidator.java index c1485cfb1a..8033688787 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AdminSettingsDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AdminSettingsDataValidator.java @@ -39,13 +39,14 @@ public class AdminSettingsDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, AdminSettings adminSettings) { + protected AdminSettings validateUpdate(TenantId tenantId, AdminSettings adminSettings) { AdminSettings existentAdminSettings = adminSettingsService.findAdminSettingsById(tenantId, adminSettings.getId()); if (existentAdminSettings != null) { if (!existentAdminSettings.getKey().equals(adminSettings.getKey())) { throw new DataValidationException("Changing key of admin settings entry is prohibited!"); } } + return existentAdminSettings; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java index 57d39ee781..a2204491f9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.BaseAssetService; -import org.thingsboard.server.dao.cache.EntitiesCacheManager; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; @@ -53,9 +52,6 @@ public class AssetDataValidator extends DataValidator { @Lazy private TbTenantProfileCache tenantProfileCache; - @Autowired - private EntitiesCacheManager cacheManager; - @Override protected void validateCreate(TenantId tenantId, Asset asset) { DefaultTenantProfileConfiguration profileConfiguration = @@ -67,14 +63,12 @@ public class AssetDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, Asset asset) { + protected Asset validateUpdate(TenantId tenantId, Asset asset) { Asset old = assetDao.findById(asset.getTenantId(), asset.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing asset!"); } - if (!old.getName().equals(asset.getName())) { - cacheManager.removeAssetFromCacheByName(tenantId, old.getName()); - } + return old; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ClientRegistrationTemplateDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ClientRegistrationTemplateDataValidator.java index d558c19e9e..b5d0247e28 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ClientRegistrationTemplateDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ClientRegistrationTemplateDataValidator.java @@ -30,7 +30,8 @@ public class ClientRegistrationTemplateDataValidator extends DataValidator { @@ -59,14 +61,16 @@ public class CustomerDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, Customer customer) { - customerDao.findCustomersByTenantIdAndTitle(customer.getTenantId().getId(), customer.getTitle()).ifPresent( + protected Customer validateUpdate(TenantId tenantId, Customer customer) { + Optional customerOpt = customerDao.findCustomersByTenantIdAndTitle(customer.getTenantId().getId(), customer.getTitle()); + customerOpt.ifPresent( c -> { if (!c.getId().equals(customer.getId())) { throw new DataValidationException("Customer with such title already exists!"); } } ); + return customerOpt.orElse(null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceCredentialsDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceCredentialsDataValidator.java index 5fca062d46..cc7e488df7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceCredentialsDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceCredentialsDataValidator.java @@ -46,7 +46,7 @@ public class DeviceCredentialsDataValidator extends DataValidator { @Autowired private OtaPackageService otaPackageService; - @Autowired - private EntitiesCacheManager cacheManager; - @Override protected void validateCreate(TenantId tenantId, Device device) { DefaultTenantProfileConfiguration profileConfiguration = @@ -73,15 +69,12 @@ public class DeviceDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, Device device) { + protected Device validateUpdate(TenantId tenantId, Device device) { Device old = deviceDao.findById(device.getTenantId(), device.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing device!"); } - if (!old.getName().equals(device.getName())) { - cacheManager.removeDeviceFromCacheByName(tenantId, old.getName()); - cacheManager.removeDeviceFromCacheById(tenantId, device.getId()); - } + return old; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index adc8cf9fb1..e67d4f9b35 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -236,7 +236,7 @@ public class DeviceProfileDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, DeviceProfile deviceProfile) { + protected DeviceProfile validateUpdate(TenantId tenantId, DeviceProfile deviceProfile) { DeviceProfile old = deviceProfileDao.findById(deviceProfile.getTenantId(), deviceProfile.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing device profile!"); @@ -255,6 +255,7 @@ public class DeviceProfileDataValidator extends DataValidator { throw new DataValidationException(message); } } + return old; } private void validateProtoSchemas(ProtoTransportPayloadConfiguration protoTransportPayloadTypeConfiguration) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java index be01ec5c9b..8289d21b23 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.cache.EntitiesCacheManager; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.edge.EdgeDao; import org.thingsboard.server.dao.exception.DataValidationException; @@ -39,18 +38,14 @@ public class EdgeDataValidator extends DataValidator { private final EdgeDao edgeDao; private final TenantDao tenantDao; private final CustomerDao customerDao; - private final EntitiesCacheManager cacheManager; @Override protected void validateCreate(TenantId tenantId, Edge edge) { } @Override - protected void validateUpdate(TenantId tenantId, Edge edge) { - Edge old = edgeDao.findById(edge.getTenantId(), edge.getId().getId()); - if (!old.getName().equals(edge.getName())) { - cacheManager.removeEdgeFromCacheByName(tenantId, old.getName()); - } + protected Edge validateUpdate(TenantId tenantId, Edge edge) { + return edgeDao.findById(edge.getTenantId(), edge.getId().getId()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java index a62e1afc9f..a38850c63c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java @@ -48,13 +48,14 @@ public class EntityViewDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, EntityView entityView) { - entityViewDao.findEntityViewByTenantIdAndName(entityView.getTenantId().getId(), entityView.getName()) - .ifPresent(e -> { - if (!e.getUuidId().equals(entityView.getUuidId())) { - throw new DataValidationException("Entity view with such name already exists!"); - } - }); + protected EntityView validateUpdate(TenantId tenantId, EntityView entityView) { + var opt = entityViewDao.findEntityViewByTenantIdAndName(entityView.getTenantId().getId(), entityView.getName()); + opt.ifPresent(e -> { + if (!e.getUuidId().equals(entityView.getUuidId())) { + throw new DataValidationException("Entity view with such name already exists!"); + } + }); + return opt.orElse(null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java index 8b261ad226..90db02d45e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java @@ -86,7 +86,7 @@ public class OtaPackageDataValidator extends BaseOtaPackageDataValidator { } @Override - protected void validateUpdate(TenantId tenantId, Tenant tenant) { + protected Tenant validateUpdate(TenantId tenantId, Tenant tenant) { Tenant old = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); if (old == null) { throw new DataValidationException("Can't update non existing tenant!"); } validateTenantProfile(tenantId, tenant); + return old; } private void validateTenantProfile(TenantId tenantId, Tenant tenant) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java index e8207534be..4c5e300966 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java @@ -56,7 +56,7 @@ public class TenantProfileDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, TenantProfile tenantProfile) { + protected TenantProfile validateUpdate(TenantId tenantId, TenantProfile tenantProfile) { TenantProfile old = tenantProfileDao.findById(TenantId.SYS_TENANT_ID, tenantProfile.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing tenant profile!"); @@ -65,5 +65,6 @@ public class TenantProfileDataValidator extends DataValidator { } else if (old.isIsolatedTbCore() != tenantProfile.isIsolatedTbCore()) { throw new DataValidationException("Can't update isolatedTbCore property!"); } + return old; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java index ec3be14866..93a1766546 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java @@ -83,8 +83,8 @@ public class WidgetTypeDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, WidgetTypeDetails widgetTypeDetails) { - WidgetType storedWidgetType = widgetTypeDao.findById(tenantId, widgetTypeDetails.getId().getId()); + protected WidgetTypeDetails validateUpdate(TenantId tenantId, WidgetTypeDetails widgetTypeDetails) { + WidgetTypeDetails storedWidgetType = widgetTypeDao.findById(tenantId, widgetTypeDetails.getId().getId()); if (!storedWidgetType.getTenantId().getId().equals(widgetTypeDetails.getTenantId().getId())) { throw new DataValidationException("Can't move existing widget type to different tenant!"); } @@ -94,5 +94,6 @@ public class WidgetTypeDataValidator extends DataValidator { if (!storedWidgetType.getAlias().equals(widgetTypeDetails.getAlias())) { throw new DataValidationException("Update of widget type alias is prohibited!"); } + return new WidgetTypeDetails(storedWidgetType); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java index d5b1a1149c..b140dfc79d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java @@ -69,7 +69,7 @@ public class WidgetsBundleDataValidator extends DataValidator { } @Override - protected void validateUpdate(TenantId tenantId, WidgetsBundle widgetsBundle) { + protected WidgetsBundle validateUpdate(TenantId tenantId, WidgetsBundle widgetsBundle) { WidgetsBundle storedWidgetsBundle = widgetsBundleDao.findById(tenantId, widgetsBundle.getId().getId()); if (!storedWidgetsBundle.getTenantId().getId().equals(widgetsBundle.getTenantId().getId())) { throw new DataValidationException("Can't move existing widgets bundle to different tenant!"); @@ -77,5 +77,6 @@ public class WidgetsBundleDataValidator extends DataValidator { if (!storedWidgetsBundle.getAlias().equals(widgetsBundle.getAlias())) { throw new DataValidationException("Update of widgets bundle alias is prohibited!"); } + return storedWidgetsBundle; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java index 94ce9efd20..115def0492 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueue.java @@ -66,7 +66,10 @@ public class TbSqlBlockingQueue implements TbSqlQueue { } queue.drainTo(entities, batchSize - 1); boolean fullPack = entities.size() == batchSize; - log.debug("[{}] Going to save {} entities", logName, entities.size()); + if (log.isDebugEnabled()) { + log.debug("[{}] Going to save {} entities", logName, entities.size()); + log.trace("[{}] Going to save entities: {}", logName, entities); + } Stream entitiesStream = entities.stream().map(TbSqlQueueElement::getEntity); saveFunction.accept( (params.isBatchSortEnabled() ? entitiesStream.sorted(batchUpdateComparator) : entitiesStream) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java index 378ce33372..be42ee3fd9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlBlockingQueueParams.java @@ -18,8 +18,6 @@ package org.thingsboard.server.dao.sql; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.stats.MessagesStats; -import org.thingsboard.server.common.stats.StatsFactory; @Slf4j @Data diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java index 0de2d839ed..db15026819 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/TbSqlQueueElement.java @@ -17,7 +17,9 @@ package org.thingsboard.server.dao.sql; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; +import lombok.ToString; +@ToString(exclude = "future") public final class TbSqlQueueElement { @Getter private final SettableFuture future; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 33513fa252..472652e357 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -28,7 +28,6 @@ import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import java.sql.PreparedStatement; import java.sql.SQLException; -import java.sql.SQLType; import java.sql.Types; import java.util.ArrayList; import java.util.List; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 20b9e0d1f7..169ef55fbe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.attributes; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -39,6 +40,7 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.List; @@ -108,33 +110,30 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String attributeType, String attributeKey) { + public Optional find(TenantId tenantId, EntityId entityId, String attributeType, String attributeKey) { AttributeKvCompositeKey compositeKey = getAttributeKvCompositeKey(entityId, attributeType, attributeKey); - return Futures.immediateFuture( - Optional.ofNullable(DaoUtil.getData(attributeKvRepository.findById(compositeKey)))); + return Optional.ofNullable(DaoUtil.getData(attributeKvRepository.findById(compositeKey))); } @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String attributeType, Collection attributeKeys) { + public List find(TenantId tenantId, EntityId entityId, String attributeType, Collection attributeKeys) { List compositeKeys = attributeKeys .stream() .map(attributeKey -> getAttributeKvCompositeKey(entityId, attributeType, attributeKey)) .collect(Collectors.toList()); - return Futures.immediateFuture( - DaoUtil.convertDataList(Lists.newArrayList(attributeKvRepository.findAllById(compositeKeys)))); + return DaoUtil.convertDataList(Lists.newArrayList(attributeKvRepository.findAllById(compositeKeys))); } @Override - public ListenableFuture> findAll(TenantId tenantId, EntityId entityId, String attributeType) { - return Futures.immediateFuture( - DaoUtil.convertDataList(Lists.newArrayList( + public List findAll(TenantId tenantId, EntityId entityId, String attributeType) { + return DaoUtil.convertDataList(Lists.newArrayList( attributeKvRepository.findAllByEntityTypeAndEntityIdAndAttributeType( entityId.getEntityType(), entityId.getId(), - attributeType)))); + attributeType))); } @Override @@ -153,7 +152,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, String attributeType, AttributeKvEntry attribute) { AttributeKvEntity entity = new AttributeKvEntity(); entity.setId(new AttributeKvCompositeKey(entityId.getEntityType(), entityId.getId(), attributeType, attribute.getKey())); entity.setLastUpdateTs(attribute.getLastUpdateTs()); @@ -165,18 +164,20 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return addToQueue(entity); } - private ListenableFuture addToQueue(AttributeKvEntity entity) { - return queue.add(entity); + private ListenableFuture addToQueue(AttributeKvEntity entity) { + return Futures.transform(queue.add(entity), v -> entity.getId().getAttributeKey(), MoreExecutors.directExecutor()); } @Override - public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List keys) { - return service.submit(() -> { - keys.forEach(key -> - attributeKvRepository.delete(entityId.getEntityType(), entityId.getId(), attributeType, key) - ); - return null; - }); + public List> removeAll(TenantId tenantId, EntityId entityId, String attributeType, List keys) { + List> futuresList = new ArrayList<>(keys.size()); + for (String key : keys) { + futuresList.add(service.submit(() -> { + attributeKvRepository.delete(entityId.getEntityType(), entityId.getId(), attributeType, key); + return key; + })); + } + return futuresList; } private AttributeKvCompositeKey getAttributeKvCompositeKey(EntityId entityId, String attributeType, String attributeKey) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index 3ecc19245d..e61a60b5b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -19,7 +19,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 8c515f13c0..28bfe52e75 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -157,9 +157,9 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao> findEntityViewsByTenantIdAndEntityIdAsync(UUID tenantId, UUID entityId) { - return service.submit(() -> DaoUtil.convertDataList( - entityViewRepository.findAllByTenantIdAndEntityId(tenantId, entityId))); + public List findEntityViewsByTenantIdAndEntityId(UUID tenantId, UUID entityId) { + return DaoUtil.convertDataList( + entityViewRepository.findAllByTenantIdAndEntityId(tenantId, entityId)); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java index f6e6dbc5c4..53c1318948 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java @@ -21,8 +21,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.model.sql.OtaPackageEntity; +import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java index 8ea805332e..ab8528c3ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java @@ -20,15 +20,15 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.OtaPackageInfo; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.ota.OtaPackageInfoDao; import org.thingsboard.server.dao.model.sql.OtaPackageInfoEntity; +import org.thingsboard.server.dao.ota.OtaPackageInfoDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.Objects; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmDataAdapter.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmDataAdapter.java index 02b4c17d7e..7d660519eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmDataAdapter.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmDataAdapter.java @@ -36,7 +36,6 @@ import org.thingsboard.server.dao.model.ModelConstants; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java index a88a2dc0d6..616d9811b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java @@ -15,12 +15,10 @@ */ package org.thingsboard.server.dao.sql.query; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmData; -import org.thingsboard.server.common.data.query.AlarmDataPageLink; import org.thingsboard.server.common.data.query.AlarmDataQuery; import java.util.Collection; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 1aaf590afd..a1be8884ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java index 2de77a0e73..49640c1640 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.query; import org.apache.commons.lang3.math.NumberUtils; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.PageData; @@ -27,7 +26,6 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.TsValue; -import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java index d4811de363..72da81786a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java @@ -17,18 +17,10 @@ package org.thingsboard.server.dao.sql.query; import lombok.AllArgsConstructor; import lombok.Getter; -import org.hibernate.type.PostgresUUIDType; -import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import java.sql.Types; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - @AllArgsConstructor public class QuerySecurityContext { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 78a94bfaa3..71e36fcae1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -19,11 +19,9 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.ConcurrencyFailureException; -import org.springframework.data.domain.PageRequest; import org.springframework.dao.DataAccessException; -import org.springframework.data.jpa.domain.Specification; +import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -35,8 +33,6 @@ import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import javax.persistence.criteria.Predicate; -import java.util.ArrayList; import java.util.List; /** @@ -52,11 +48,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple @Autowired private RelationInsertRepository relationInsertRepository; - @Override - public ListenableFuture> findAllByFromAsync(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup) { - return service.submit(() -> findAllByFrom(tenantId, from, typeGroup)); - } - @Override public List findAllByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -67,18 +58,13 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public ListenableFuture> findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup) { - return service.submit(() -> DaoUtil.convertDataList( + public List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup) { + return DaoUtil.convertDataList( relationRepository.findAllByFromIdAndFromTypeAndRelationTypeAndRelationTypeGroup( from.getId(), from.getEntityType().name(), relationType, - typeGroup.name()))); - } - - @Override - public ListenableFuture> findAllByToAsync(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup) { - return service.submit(() -> findAllByTo(tenantId, to, typeGroup)); + typeGroup.name())); } @Override @@ -91,13 +77,13 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public ListenableFuture> findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { - return service.submit(() -> DaoUtil.convertDataList( + public List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { + return DaoUtil.convertDataList( relationRepository.findAllByToIdAndToTypeAndRelationTypeAndRelationTypeGroup( to.getId(), to.getEntityType().name(), relationType, - typeGroup.name()))); + typeGroup.name())); } @Override @@ -107,9 +93,9 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public ListenableFuture getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); - return service.submit(() -> DaoUtil.getData(relationRepository.findById(key))); + return DaoUtil.getData(relationRepository.findById(key)); } private RelationCompositeKey getRelationCompositeKey(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 64895a7f5a..f47b658601 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.thingsboard.server.common.data.id.EntityId; @@ -42,7 +41,11 @@ import org.thingsboard.server.dao.timeseries.TimeseriesDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.*; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index c938f11a83..9c03a9e3dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -21,7 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; @@ -34,7 +33,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; @@ -46,10 +44,12 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.sql.CallableStatement; -import java.sql.SQLException; -import java.sql.Types; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Function; diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileCacheKey.java new file mode 100644 index 0000000000..a9bfcbe2ca --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileCacheKey.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantProfileId; + +import java.io.Serializable; + +@Data +public class TenantProfileCacheKey implements Serializable { + + private static final long serialVersionUID = 8220455917177676472L; + + private final TenantProfileId tenantProfileId; + private final boolean defaultProfile; + + private TenantProfileCacheKey(TenantProfileId tenantProfileId, boolean defaultProfile) { + this.tenantProfileId = tenantProfileId; + this.defaultProfile = defaultProfile; + } + + public static TenantProfileCacheKey fromId(TenantProfileId id) { + return new TenantProfileCacheKey(id, false); + } + + public static TenantProfileCacheKey defaultProfile() { + return new TenantProfileCacheKey(null, true); + } + + + @Override + public String toString() { + if (defaultProfile) { + return "default"; + } else { + return tenantProfileId.toString(); + } + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileCaffeineCache.java new file mode 100644 index 0000000000..aca7a63d48 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("TenantProfileCache") +public class TenantProfileCaffeineCache extends CaffeineTbTransactionalCache { + + public TenantProfileCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.TENANT_PROFILE_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileEvictEvent.java new file mode 100644 index 0000000000..be5e2cf64d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileEvictEvent.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantProfileId; + +@Data +public class TenantProfileEvictEvent { + private final TenantProfileId tenantProfileId; + private final boolean defaultProfile; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileRedisCache.java new file mode 100644 index 0000000000..da83473f44 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TbRedisSerializer; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("TenantProfileCache") +public class TenantProfileRedisCache extends RedisTbTransactionalCache { + + public TenantProfileRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.TENANT_PROFILE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 0976c49410..71140140c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -18,10 +18,10 @@ package org.thingsboard.server.dao.tenant; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; -import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantId; @@ -30,47 +30,55 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; -import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; -import java.util.Arrays; -import java.util.Collections; +import java.util.ArrayList; +import java.util.List; -import static org.thingsboard.server.common.data.CacheConstants.TENANT_PROFILE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @Service @Slf4j -public class TenantProfileServiceImpl extends AbstractEntityService implements TenantProfileService { +public class TenantProfileServiceImpl extends AbstractCachedEntityService implements TenantProfileService { private static final String INCORRECT_TENANT_PROFILE_ID = "Incorrect tenantProfileId "; @Autowired private TenantProfileDao tenantProfileDao; - @Autowired - private CacheManager cacheManager; - @Autowired private DataValidator tenantProfileValidator; - @Cacheable(cacheNames = TENANT_PROFILE_CACHE, key = "{#tenantProfileId.id}") + @TransactionalEventListener(classes = TenantProfileEvictEvent.class) + @Override + public void handleEvictEvent(TenantProfileEvictEvent event) { + List keys = new ArrayList<>(2); + if (event.getTenantProfileId() != null) { + keys.add(TenantProfileCacheKey.fromId(event.getTenantProfileId())); + } + if (event.isDefaultProfile()) { + keys.add(TenantProfileCacheKey.defaultProfile()); + } + cache.evict(keys); + } + @Override public TenantProfile findTenantProfileById(TenantId tenantId, TenantProfileId tenantProfileId) { log.trace("Executing findTenantProfileById [{}]", tenantProfileId); Validator.validateId(tenantProfileId, INCORRECT_TENANT_PROFILE_ID + tenantProfileId); - return tenantProfileDao.findById(tenantId, tenantProfileId.getId()); + return cache.getAndPutInTransaction(TenantProfileCacheKey.fromId(tenantProfileId), + () -> tenantProfileDao.findById(tenantId, tenantProfileId.getId()), true); } - @Cacheable(cacheNames = TENANT_PROFILE_CACHE, key = "{'info', #tenantProfileId.id}") @Override public EntityInfo findTenantProfileInfoById(TenantId tenantId, TenantProfileId tenantProfileId) { log.trace("Executing findTenantProfileInfoById [{}]", tenantProfileId); - Validator.validateId(tenantProfileId, INCORRECT_TENANT_PROFILE_ID + tenantProfileId); - return tenantProfileDao.findTenantProfileInfoById(tenantId, tenantProfileId.getId()); + TenantProfile profile = findTenantProfileById(tenantId, tenantProfileId); + return profile == null ? null : new EntityInfo(profile.getId(), profile.getName()); } @Override @@ -80,7 +88,9 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T TenantProfile savedTenantProfile; try { savedTenantProfile = tenantProfileDao.save(tenantId, tenantProfile); + publishEvictEvent(new TenantProfileEvictEvent(savedTenantProfile.getId(), savedTenantProfile.isDefault())); } catch (Exception t) { + handleEvictEvent(new TenantProfileEvictEvent(null, tenantProfile.isDefault())); ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("tenant_profile_name_unq_key")) { throw new DataValidationException("Tenant profile with such name already exists!"); @@ -88,13 +98,6 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T throw t; } } - Cache cache = cacheManager.getCache(TENANT_PROFILE_CACHE); - cache.evict(Collections.singletonList(savedTenantProfile.getId().getId())); - cache.evict(Arrays.asList("info", savedTenantProfile.getId().getId())); - if (savedTenantProfile.isDefault()) { - cache.evict(Collections.singletonList("default")); - cache.evict(Arrays.asList("default", "info")); - } return savedTenantProfile; } @@ -121,13 +124,7 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T } } deleteEntityRelations(tenantId, tenantProfileId); - Cache cache = cacheManager.getCache(TENANT_PROFILE_CACHE); - cache.evict(Collections.singletonList(tenantProfileId.getId())); - cache.evict(Arrays.asList("info", tenantProfileId.getId())); - if (isDefault) { - cache.evict(Collections.singletonList("default")); - cache.evict(Arrays.asList("default", "info")); - } + publishEvictEvent(new TenantProfileEvictEvent(tenantProfileId, isDefault)); } @Override @@ -163,18 +160,19 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T return defaultTenantProfile; } - @Cacheable(cacheNames = TENANT_PROFILE_CACHE, key = "{'default'}") @Override public TenantProfile findDefaultTenantProfile(TenantId tenantId) { log.trace("Executing findDefaultTenantProfile"); - return tenantProfileDao.findDefaultTenantProfile(tenantId); + return cache.getAndPutInTransaction(TenantProfileCacheKey.defaultProfile(), + () -> tenantProfileDao.findDefaultTenantProfile(tenantId), true); + } - @Cacheable(cacheNames = TENANT_PROFILE_CACHE, key = "{'default', 'info'}") @Override public EntityInfo findDefaultTenantProfileInfo(TenantId tenantId) { log.trace("Executing findDefaultTenantProfileInfo"); - return tenantProfileDao.findDefaultTenantProfileInfo(tenantId); + var tenantProfile = findDefaultTenantProfile(tenantId); + return tenantProfile == null ? null : new EntityInfo(tenantProfile.getId(), tenantProfile.getName()); } @Override @@ -183,27 +181,21 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T validateId(tenantId, INCORRECT_TENANT_PROFILE_ID + tenantProfileId); TenantProfile tenantProfile = tenantProfileDao.findById(tenantId, tenantProfileId.getId()); if (!tenantProfile.isDefault()) { - Cache cache = cacheManager.getCache(TENANT_PROFILE_CACHE); tenantProfile.setDefault(true); TenantProfile previousDefaultTenantProfile = findDefaultTenantProfile(tenantId); boolean changed = false; if (previousDefaultTenantProfile == null) { tenantProfileDao.save(tenantId, tenantProfile); + publishEvictEvent(new TenantProfileEvictEvent(tenantProfileId, true)); changed = true; } else if (!previousDefaultTenantProfile.getId().equals(tenantProfile.getId())) { previousDefaultTenantProfile.setDefault(false); tenantProfileDao.save(tenantId, previousDefaultTenantProfile); tenantProfileDao.save(tenantId, tenantProfile); - cache.evict(Collections.singletonList(previousDefaultTenantProfile.getId().getId())); - cache.evict(Arrays.asList("info", previousDefaultTenantProfile.getId().getId())); + publishEvictEvent(new TenantProfileEvictEvent(previousDefaultTenantProfile.getId(), false)); + publishEvictEvent(new TenantProfileEvictEvent(tenantProfileId, true)); changed = true; } - if (changed) { - cache.evict(Collections.singletonList(tenantProfile.getId().getId())); - cache.evict(Arrays.asList("info", tenantProfile.getId().getId())); - cache.evict(Collections.singletonList("default")); - cache.evict(Arrays.asList("default", "info")); - } return changed; } return false; @@ -216,7 +208,7 @@ public class TenantProfileServiceImpl extends AbstractEntityService implements T } private final PaginatedRemover tenantProfilesRemover = - new PaginatedRemover() { + new PaginatedRemover<>() { @Override protected PageData findEntities(TenantId tenantId, String id, PageLink pageLink) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 0d71e3f716..a74f51ce9e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -36,8 +36,8 @@ import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.Validator; diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 79a3136589..124af03a1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -20,7 +20,6 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -34,18 +33,16 @@ import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; -import javax.annotation.Nullable; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index c3705fbc3d..d24a229766 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -20,8 +20,8 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; -import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import java.util.List; diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 2193305837..db52629d8d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.dao.usagerecord; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ApiFeature; import org.thingsboard.server.common.data.ApiUsageRecordKey; @@ -47,7 +47,6 @@ import static org.thingsboard.server.dao.service.Validator.validateId; @Service @Slf4j -@AllArgsConstructor public class ApiUsageStateServiceImpl extends AbstractEntityService implements ApiUsageStateService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -57,6 +56,16 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A private final TimeseriesService tsService; private final DataValidator apiUsageStateValidator; + public ApiUsageStateServiceImpl(ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, + TenantDao tenantDao, @Lazy TimeseriesService tsService, + DataValidator apiUsageStateValidator) { + this.apiUsageStateDao = apiUsageStateDao; + this.tenantProfileDao = tenantProfileDao; + this.tenantDao = tenantDao; + this.tsService = tsService; + this.apiUsageStateValidator = apiUsageStateValidator; + } + @Override public void deleteApiUsageStateByTenantId(TenantId tenantId) { log.trace("Executing deleteUsageRecordsByTenantId [{}]", tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index ba0f09d64e..6ce166be71 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; -import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Value; diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java new file mode 100644 index 0000000000..d54933a19d --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.service.DaoSqlTest; + +@RunWith(SpringRunner.class) +@ContextConfiguration(classes = {JpaDaoConfig.class, PsqlTsDaoConfig.class, PsqlTsLatestDaoConfig.class, SqlTimeseriesDaoConfig.class}) +@DaoSqlTest +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@TestExecutionListeners({ + DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class}) +public abstract class AbstractDaoServiceTest { + + @MockBean(answer = Answers.RETURNS_MOCKS) + StatsFactory statsFactory; + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/CustomSqlUnit.java b/dao/src/test/java/org/thingsboard/server/dao/CustomSqlUnit.java deleted file mode 100644 index 73e12f6e78..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/CustomSqlUnit.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import com.google.common.base.Charsets; -import com.google.common.io.Resources; -import lombok.extern.slf4j.Slf4j; -import org.junit.rules.ExternalResource; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.List; -import java.util.Properties; - - -/** - * Created by Valerii Sosliuk on 6/24/2017. - * - * Deprecated. Use PostgreSqlInitializer class instead - */ -@Slf4j -@Deprecated -public class CustomSqlUnit extends ExternalResource { - - private final List sqlFiles; - private final String dropAllTablesSqlFile; - private final String dbUrl; - private final String dbUserName; - private final String dbPassword; - - public CustomSqlUnit(List sqlFiles, String dropAllTablesSqlFile, String configurationFileName) { - this.sqlFiles = sqlFiles; - this.dropAllTablesSqlFile = dropAllTablesSqlFile; - final Properties properties = new Properties(); - try (final InputStream stream = this.getClass().getClassLoader().getResourceAsStream(configurationFileName)) { - properties.load(stream); - this.dbUrl = properties.getProperty("spring.datasource.url"); - this.dbUserName = properties.getProperty("spring.datasource.username"); - this.dbPassword = properties.getProperty("spring.datasource.password"); - } catch (IOException e) { - throw new RuntimeException(e.getMessage(), e); - } - } - - @Override - public void before() { - cleanUpDb(); - - Connection conn = null; - try { - conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword); - for (String sqlFile : sqlFiles) { - URL sqlFileUrl = Resources.getResource(sqlFile); - String sql = Resources.toString(sqlFileUrl, Charsets.UTF_8); - conn.createStatement().execute(sql); - } - } catch (IOException | SQLException e) { - throw new RuntimeException("Unable to start embedded hsqldb. Reason: " + e.getMessage(), e); - } finally { - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - log.error(e.getMessage(), e); - } - } - } - } - - @Override - public void after() { - cleanUpDb(); - } - - private void cleanUpDb() { - Connection conn = null; - try { - conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword); - URL dropAllTableSqlFileUrl = Resources.getResource(dropAllTablesSqlFile); - String dropAllTablesSql = Resources.toString(dropAllTableSqlFileUrl, Charsets.UTF_8); - conn.createStatement().execute(dropAllTablesSql); - } catch (IOException | SQLException e) { - throw new RuntimeException("Unable to clean up embedded hsqldb. Reason: " + e.getMessage(), e); - } finally { - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - log.error(e.getMessage(), e); - } - } - } - } -} diff --git a/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java b/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java deleted file mode 100644 index 744d529897..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import com.github.springtestdbunit.bean.DatabaseConfigBean; -import com.github.springtestdbunit.bean.DatabaseDataSourceConnectionFactoryBean; -import org.dbunit.DatabaseUnitException; -import org.dbunit.ext.hsqldb.HsqldbDataTypeFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import javax.sql.DataSource; -import java.io.IOException; -import java.sql.SQLException; - -/** - * Created by Valerii Sosliuk on 5/6/2017. - */ -@Configuration -@Deprecated -public class JpaDbunitTestConfig { - - @Autowired - private DataSource dataSource; - - @Bean - public DatabaseConfigBean databaseConfigBean() { - DatabaseConfigBean databaseConfigBean = new DatabaseConfigBean(); - databaseConfigBean.setDatatypeFactory(new HsqldbDataTypeFactory()); - return databaseConfigBean; - } - - @Bean(name = "dbUnitDatabaseConnection") - public DatabaseDataSourceConnectionFactoryBean dbUnitDatabaseConnection() throws SQLException, DatabaseUnitException, IOException { - DatabaseDataSourceConnectionFactoryBean databaseDataSourceConnectionFactoryBean = new DatabaseDataSourceConnectionFactoryBean(); - databaseDataSourceConnectionFactoryBean.setDatabaseConfig(databaseConfigBean()); - databaseDataSourceConnectionFactoryBean.setDataSource(dataSource); - return databaseDataSourceConnectionFactoryBean; - } -} diff --git a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java index 569535a535..838be51349 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java @@ -25,7 +25,7 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClassnameFilters({ - "org.thingsboard.server.dao.service.nosql.*ServiceNoSqlTest", + "org.thingsboard.server.dao.service.*.nosql.*ServiceNoSqlTest", }) public class NoSqlDaoServiceTestSuite { diff --git a/dao/src/test/java/org/thingsboard/server/dao/RedisSqlTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/RedisSqlTestSuite.java new file mode 100644 index 0000000000..01871e5d3b --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/RedisSqlTestSuite.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.junit.ClassRule; +import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; +import org.junit.runner.RunWith; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.support.TestPropertySourceUtils; +import org.testcontainers.containers.GenericContainer; + +@ContextConfiguration(initializers = RedisSqlTestSuite.class) +@RunWith(ClasspathSuite.class) +@ClassnameFilters( + //All the same tests using redis instead of caffeine. + "org.thingsboard.server.dao.service.*ServiceSqlTest" +) +public class RedisSqlTestSuite implements ApplicationContextInitializer { + + @ClassRule + public static GenericContainer redis = new GenericContainer("redis:4.0").withExposedPorts(6379); + + @Override + public void initialize(ConfigurableApplicationContext applicationContext) { + TestPropertySourceUtils.addInlinedPropertiesToEnvironment( + applicationContext, "cache.type=redis"); + TestPropertySourceUtils.addInlinedPropertiesToEnvironment( + applicationContext, "redis.connection.type=standalone"); + TestPropertySourceUtils.addInlinedPropertiesToEnvironment( + applicationContext, "redis.standalone.host=localhost"); + TestPropertySourceUtils.addInlinedPropertiesToEnvironment( + applicationContext, "redis.standalone.port=" + redis.getMappedPort(6379)); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java deleted file mode 100644 index 12f77ece14..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters; -import org.junit.runner.RunWith; - -@RunWith(ClasspathSuite.class) -@ClassnameFilters({ - "org.thingsboard.server.dao.service.attributes.sql.*SqlTest", - "org.thingsboard.server.dao.service.event.sql.*SqlTest", - "org.thingsboard.server.dao.service.sql.*SqlTest", - "org.thingsboard.server.dao.service.timeseries.sql.*SqlTest", - "org.thingsboard.server.dao.service.install.sql.*SqlTest" -}) -public class SqlDaoServiceTestSuite { -} diff --git a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java deleted file mode 100644 index a45eb0b711..0000000000 --- a/dao/src/test/java/org/thingsboard/server/dao/attributes/CachedAttributesServiceTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.attributes; - -import com.google.common.util.concurrent.MoreExecutors; -import org.junit.Test; -import org.thingsboard.server.dao.cache.CacheExecutorService; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.willCallRealMethod; -import static org.mockito.Mockito.mock; - -public class CachedAttributesServiceTest { - - public static final String REDIS = "redis"; - - @Test - public void givenLocalCacheTypeName_whenEquals_thenOK() { - assertThat(CachedAttributesService.LOCAL_CACHE_TYPE, is("caffeine")); - } - - @Test - public void givenCacheType_whenGetExecutor_thenDirectExecutor() { - CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); - CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); - willCallRealMethod().given(cachedAttributesService).getExecutor(any(), any()); - - assertThat(cachedAttributesService.getExecutor(null, cacheExecutorService), is(MoreExecutors.directExecutor())); - - assertThat(cachedAttributesService.getExecutor("", cacheExecutorService), is(MoreExecutors.directExecutor())); - - assertThat(cachedAttributesService.getExecutor(CachedAttributesService.LOCAL_CACHE_TYPE, cacheExecutorService), is(MoreExecutors.directExecutor())); - - } - - @Test - public void givenCacheType_whenGetExecutor_thenReturnCacheExecutorService() { - CachedAttributesService cachedAttributesService = mock(CachedAttributesService.class); - CacheExecutorService cacheExecutorService = mock(CacheExecutorService.class); - willCallRealMethod().given(cachedAttributesService).getExecutor(any(String.class), any(CacheExecutorService.class)); - - assertThat(cachedAttributesService.getExecutor(REDIS, cacheExecutorService), is(cacheExecutorService)); - - assertThat(cachedAttributesService.getExecutor("unknownCacheType", cacheExecutorService), is(cacheExecutorService)); - - } - -} \ No newline at end of file diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java index 689752f0fe..3bbe6f22ab 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceCredentialsCacheTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceCredentialsDao; import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceDao; import org.thingsboard.server.dao.device.DeviceService; import java.util.UUID; @@ -70,7 +71,6 @@ public abstract class BaseDeviceCredentialsCacheTest extends AbstractServiceTest ReflectionTestUtils.setField(unwrapDeviceCredentialsService(), "deviceCredentialsDao", deviceCredentialsDao); ReflectionTestUtils.setField(unwrapDeviceCredentialsService(), "credentialsValidator", credentialsValidator); - } @After @@ -120,7 +120,10 @@ public abstract class BaseDeviceCredentialsCacheTest extends AbstractServiceTest when(deviceCredentialsDao.findById(SYSTEM_TENANT_ID, deviceCredentialsId)).thenReturn(createDummyDeviceCredentialsEntity(CREDENTIALS_ID_1)); when(deviceService.findDeviceById(SYSTEM_TENANT_ID, new DeviceId(deviceId))).thenReturn(new Device()); - deviceCredentialsService.updateDeviceCredentials(SYSTEM_TENANT_ID, createDummyDeviceCredentials(deviceCredentialsId, CREDENTIALS_ID_2, deviceId)); + var dummy = createDummyDeviceCredentials(deviceCredentialsId, CREDENTIALS_ID_2, deviceId); + when(deviceCredentialsDao.saveAndFlush(SYSTEM_TENANT_ID, dummy)).thenReturn(dummy); + + deviceCredentialsService.updateDeviceCredentials(SYSTEM_TENANT_ID, dummy); when(deviceCredentialsDao.findByCredentialsId(SYSTEM_TENANT_ID, CREDENTIALS_ID_1)).thenReturn(null); @@ -135,7 +138,7 @@ public abstract class BaseDeviceCredentialsCacheTest extends AbstractServiceTest Object target = ((Advised) deviceCredentialsService).getTargetSource().getTarget(); return (DeviceCredentialsService) target; } - return null; + return deviceCredentialsService; } private DeviceCredentials createDummyDeviceCredentialsEntity(String deviceCredentialsId) { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java index 4e0b0e7dfc..393c89db5d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java @@ -43,6 +43,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java index 33cb7e1bc1..73b26df9e8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; @@ -33,14 +34,33 @@ import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import java.io.IOException; +import java.text.ParseException; import java.time.LocalDateTime; import java.time.Month; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; public abstract class BaseEdgeEventServiceTest extends AbstractServiceTest { + long timeBeforeStartTime; + long startTime; + long eventTime; + long endTime; + long timeAfterEndTime; + + @Before + public void before() throws ParseException { + timeBeforeStartTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T11:30:00Z").getTime(); + startTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T12:00:00Z").getTime(); + eventTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T12:30:00Z").getTime(); + endTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T13:00:00Z").getTime(); + timeAfterEndTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T13:30:30Z").getTime(); + } + @Test public void saveEdgeEvent() throws Exception { EdgeId edgeId = new EdgeId(Uuids.timeBased()); @@ -75,15 +95,8 @@ public abstract class BaseEdgeEventServiceTest extends AbstractServiceTest { return edgeEvent; } - @Test public void findEdgeEventsByTimeDescOrder() throws Exception { - long timeBeforeStartTime = LocalDateTime.of(2020, Month.NOVEMBER, 1, 11, 30).toEpochSecond(ZoneOffset.UTC); - long startTime = LocalDateTime.of(2020, Month.NOVEMBER, 1, 12, 0).toEpochSecond(ZoneOffset.UTC); - long eventTime = LocalDateTime.of(2020, Month.NOVEMBER, 1, 12, 30).toEpochSecond(ZoneOffset.UTC); - long endTime = LocalDateTime.of(2020, Month.NOVEMBER, 1, 13, 0).toEpochSecond(ZoneOffset.UTC); - long timeAfterEndTime = LocalDateTime.of(2020, Month.NOVEMBER, 1, 13, 30).toEpochSecond(ZoneOffset.UTC); - EdgeId edgeId = new EdgeId(Uuids.timeBased()); DeviceId deviceId = new DeviceId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); @@ -113,6 +126,8 @@ public abstract class BaseEdgeEventServiceTest extends AbstractServiceTest { Assert.assertEquals(1, edgeEvents.getData().size()); Assert.assertEquals(Uuids.startOf(eventTime), edgeEvents.getData().get(0).getUuidId()); Assert.assertFalse(edgeEvents.hasNext()); + + edgeEventService.cleanupEvents(1); } @Test diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index 430ac85391..808e499239 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -339,12 +339,12 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); } - Futures.successfulAsList(attributeFutures).get(); + Futures.allAsList(attributeFutures).get(); RelationsQueryFilter filter = new RelationsQueryFilter(); filter.setRootEntity(tenantId); @@ -518,12 +518,12 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { List highTemperatures = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, new ArrayList<>(), new ArrayList<>(), temperatures, highTemperatures); - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); } - Futures.successfulAsList(attributeFutures).get(); + Futures.allAsList(attributeFutures).get(); DeviceSearchQueryFilter filter = new DeviceSearchQueryFilter(); filter.setRootEntity(tenantId); @@ -593,12 +593,12 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { List highConsumptions = new ArrayList<>(); createTestHierarchy(tenantId, assets, devices, consumptions, highConsumptions, new ArrayList<>(), new ArrayList<>()); - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < assets.size(); i++) { Asset asset = assets.get(i); attributeFutures.add(saveLongAttribute(asset.getId(), "consumption", consumptions.get(i), DataConstants.SERVER_SCOPE)); } - Futures.successfulAsList(attributeFutures).get(); + Futures.allAsList(attributeFutures).get(); AssetSearchQueryFilter filter = new AssetSearchQueryFilter(); filter.setRootEntity(tenantId); @@ -1048,14 +1048,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } } - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); for (String currentScope : DataConstants.allScopes()) { attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), currentScope)); } } - Futures.successfulAsList(attributeFutures).get(); + Futures.allAsList(attributeFutures).get(); DeviceTypeFilter filter = new DeviceTypeFilter(); filter.setDeviceType("default"); @@ -1150,12 +1150,12 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } } - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveLongAttribute(device.getId(), "temperature", temperatures.get(i), DataConstants.CLIENT_SCOPE)); } - Futures.successfulAsList(attributeFutures).get(); + Futures.allAsList(attributeFutures).get(); DeviceTypeFilter filter = new DeviceTypeFilter(); filter.setDeviceType("default"); @@ -1301,7 +1301,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); } - Futures.successfulAsList(timeseriesFutures).get(); + Futures.allAsList(timeseriesFutures).get(); DeviceTypeFilter filter = new DeviceTypeFilter(); filter.setDeviceType("default"); @@ -1428,12 +1428,12 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { } } - List>> attributeFutures = new ArrayList<>(); + List>> attributeFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); attributeFutures.add(saveStringAttribute(device.getId(), "attributeString", attributeStrings.get(i), DataConstants.CLIENT_SCOPE)); } - Futures.successfulAsList(attributeFutures).get(); + Futures.allAsList(attributeFutures).get(); DeviceTypeFilter filter = new DeviceTypeFilter(); filter.setDeviceType("default"); @@ -1764,13 +1764,13 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { return filter; } - private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, String scope) { + private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, String scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, String scope) { + private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, String scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationCacheTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationCacheTest.java index 8023133059..4ef6df8015 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationCacheTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationCacheTest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.service; -import com.google.common.util.concurrent.Futures; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -69,13 +68,13 @@ public abstract class BaseRelationCacheTest extends AbstractServiceTest { Object target = ((Advised) relationService).getTargetSource().getTarget(); return (RelationService) target; } - return null; + return relationService; } @Test public void testFindRelationByFrom_Cached() throws ExecutionException, InterruptedException { when(relationDao.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON)) - .thenReturn(Futures.immediateFuture(new EntityRelation(ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE))); + .thenReturn(new EntityRelation(ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE)); relationService.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); relationService.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); @@ -86,7 +85,7 @@ public abstract class BaseRelationCacheTest extends AbstractServiceTest { @Test public void testDeleteRelations_EvictsCache() { when(relationDao.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON)) - .thenReturn(Futures.immediateFuture(new EntityRelation(ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE))); + .thenReturn(new EntityRelation(ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE)); relationService.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); relationService.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); @@ -99,6 +98,5 @@ public abstract class BaseRelationCacheTest extends AbstractServiceTest { relationService.getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); verify(relationDao, times(2)).getRelation(SYSTEM_TENANT_ID, ENTITY_ID_FROM, ENTITY_ID_TO, RELATION_TYPE, RelationTypeGroup.COMMON); - } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index 043db133b6..4abd33ab1a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -16,28 +16,49 @@ package org.thingsboard.server.dao.service.attributes; import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.attributes.AttributeCacheKey; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.service.AbstractServiceTest; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * @author Andrew Shvayka */ +@Slf4j public abstract class BaseAttributesServiceTest extends AbstractServiceTest { + private static final String OLD_VALUE = "OLD VALUE"; + private static final String NEW_VALUE = "NEW VALUE"; + + @Autowired + private TbTransactionalCache cache; + @Autowired private AttributesService attributesService; @@ -100,4 +121,172 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { Assert.assertEquals(attrBNew, saved.get(1)); } + @Test + public void testDummyRequestWithEmptyResult() throws Exception { + var future = attributesService.find(new TenantId(UUID.randomUUID()), new DeviceId(UUID.randomUUID()), DataConstants.SERVER_SCOPE, "TEST"); + Assert.assertNotNull(future); + var result = future.get(10, TimeUnit.SECONDS); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testConcurrentTransaction() throws Exception { + var tenantId = new TenantId(UUID.randomUUID()); + var deviceId = new DeviceId(UUID.randomUUID()); + var scope = DataConstants.SERVER_SCOPE; + var key = "TEST"; + + var attrKey = new AttributeCacheKey(scope, deviceId, "TEST"); + var oldValue = new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, OLD_VALUE)); + var newValue = new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, NEW_VALUE)); + + var trx = cache.newTransactionForKey(attrKey); + cache.putIfAbsent(attrKey, newValue); + trx.putIfAbsent(attrKey, oldValue); + Assert.assertFalse(trx.commit()); + Assert.assertEquals(NEW_VALUE, getAttributeValue(tenantId, deviceId, scope, key)); + } + + @Test + public void testConcurrentFetchAndUpdate() throws Exception { + var tenantId = new TenantId(UUID.randomUUID()); + ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2)); + try { + for (int i = 0; i < 100; i++) { + var deviceId = new DeviceId(UUID.randomUUID()); + testConcurrentFetchAndUpdate(tenantId, deviceId, pool); + } + } finally { + pool.shutdownNow(); + } + } + + @Test + public void testConcurrentFetchAndUpdateMulti() throws Exception { + var tenantId = new TenantId(UUID.randomUUID()); + ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2)); + try { + for (int i = 0; i < 100; i++) { + var deviceId = new DeviceId(UUID.randomUUID()); + testConcurrentFetchAndUpdateMulti(tenantId, deviceId, pool); + } + } finally { + pool.shutdownNow(); + } + } + + @Test + public void testFetchAndUpdateEmpty() throws Exception { + var tenantId = new TenantId(UUID.randomUUID()); + var deviceId = new DeviceId(UUID.randomUUID()); + var scope = DataConstants.SERVER_SCOPE; + var key = "TEST"; + + Optional emptyValue = attributesService.find(tenantId, deviceId, scope, key).get(10, TimeUnit.SECONDS); + Assert.assertTrue(emptyValue.isEmpty()); + + saveAttribute(tenantId, deviceId, scope, key, NEW_VALUE); + Assert.assertEquals(NEW_VALUE, getAttributeValue(tenantId, deviceId, scope, key)); + } + + @Test + public void testFetchAndUpdateMulti() throws Exception { + var tenantId = new TenantId(UUID.randomUUID()); + var deviceId = new DeviceId(UUID.randomUUID()); + var scope = DataConstants.SERVER_SCOPE; + var key1 = "TEST1"; + var key2 = "TEST2"; + + var value = getAttributeValues(tenantId, deviceId, scope, Arrays.asList(key1, key2)); + Assert.assertTrue(value.isEmpty()); + + saveAttribute(tenantId, deviceId, scope, key1, OLD_VALUE); + + value = getAttributeValues(tenantId, deviceId, scope, Arrays.asList(key1, key2)); + Assert.assertEquals(1, value.size()); + Assert.assertEquals(OLD_VALUE, value.get(0)); + + saveAttribute(tenantId, deviceId, scope, key2, NEW_VALUE); + + value = getAttributeValues(tenantId, deviceId, scope, Arrays.asList(key1, key2)); + Assert.assertEquals(2, value.size()); + Assert.assertTrue(value.contains(OLD_VALUE)); + Assert.assertTrue(value.contains(NEW_VALUE)); + + saveAttribute(tenantId, deviceId, scope, key1, NEW_VALUE); + + value = getAttributeValues(tenantId, deviceId, scope, Arrays.asList(key1, key2)); + Assert.assertEquals(2, value.size()); + Assert.assertEquals(NEW_VALUE, value.get(0)); + Assert.assertEquals(NEW_VALUE, value.get(1)); + } + + private void testConcurrentFetchAndUpdate(TenantId tenantId, DeviceId deviceId, ListeningExecutorService pool) throws Exception { + var scope = DataConstants.SERVER_SCOPE; + var key = "TEST"; + saveAttribute(tenantId, deviceId, scope, key, OLD_VALUE); + List> futures = new ArrayList<>(); + futures.add(pool.submit(() -> { + var value = getAttributeValue(tenantId, deviceId, scope, key); + Assert.assertTrue(value.equals(OLD_VALUE) || value.equals(NEW_VALUE)); + })); + futures.add(pool.submit(() -> saveAttribute(tenantId, deviceId, scope, key, NEW_VALUE))); + Futures.allAsList(futures).get(10, TimeUnit.SECONDS); + Assert.assertEquals(NEW_VALUE, getAttributeValue(tenantId, deviceId, scope, key)); + } + + private void testConcurrentFetchAndUpdateMulti(TenantId tenantId, DeviceId deviceId, ListeningExecutorService pool) throws Exception { + var scope = DataConstants.SERVER_SCOPE; + var key1 = "TEST1"; + var key2 = "TEST2"; + saveAttribute(tenantId, deviceId, scope, key1, OLD_VALUE); + saveAttribute(tenantId, deviceId, scope, key2, OLD_VALUE); + List> futures = new ArrayList<>(); + futures.add(pool.submit(() -> { + var value = getAttributeValues(tenantId, deviceId, scope, Arrays.asList(key1, key2)); + Assert.assertEquals(2, value.size()); + Assert.assertTrue(value.contains(OLD_VALUE) || value.contains(NEW_VALUE)); + })); + futures.add(pool.submit(() -> { + saveAttribute(tenantId, deviceId, scope, key1, NEW_VALUE); + saveAttribute(tenantId, deviceId, scope, key2, NEW_VALUE); + })); + Futures.allAsList(futures).get(10, TimeUnit.SECONDS); + var newResult = getAttributeValues(tenantId, deviceId, scope, Arrays.asList(key1, key2)); + Assert.assertEquals(2, newResult.size()); + Assert.assertEquals(NEW_VALUE, newResult.get(0)); + Assert.assertEquals(NEW_VALUE, newResult.get(1)); + } + + private String getAttributeValue(TenantId tenantId, DeviceId deviceId, String scope, String key) { + try { + Optional entry = attributesService.find(tenantId, deviceId, scope, key).get(10, TimeUnit.SECONDS); + return entry.orElseThrow(RuntimeException::new).getStrValue().orElse("Unknown"); + } catch (Exception e) { + log.warn("Failed to get attribute", e.getCause()); + throw new RuntimeException(e); + } + } + + private List getAttributeValues(TenantId tenantId, DeviceId deviceId, String scope, List keys) { + try { + List entry = attributesService.find(tenantId, deviceId, scope, keys).get(10, TimeUnit.SECONDS); + return entry.stream().map(e -> e.getStrValue().orElse(null)).collect(Collectors.toList()); + } catch (Exception e) { + log.warn("Failed to get attributes", e.getCause()); + throw new RuntimeException(e); + } + } + + private void saveAttribute(TenantId tenantId, DeviceId deviceId, String scope, String key, String s) { + try { + AttributeKvEntry newEntry = new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, s)); + attributesService.save(tenantId, deviceId, scope, Collections.singletonList(newEntry)).get(10, TimeUnit.SECONDS); + } catch (Exception e) { + log.warn("Failed to save attribute", e.getCause()); + Assert.assertNull(e); + } + } + + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java index 7d81e5afc6..f3f6dd8ff8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java @@ -16,8 +16,9 @@ package org.thingsboard.server.dao.service.event; import com.datastax.oss.driver.api.core.uuid.Uuids; +import org.apache.commons.lang3.time.DateFormatUtils; import org.junit.Assert; -import org.junit.Ignore; +import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Event; @@ -31,14 +32,29 @@ import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.service.AbstractServiceTest; -import java.io.IOException; +import java.text.ParseException; import java.time.LocalDateTime; import java.time.Month; import java.time.ZoneOffset; import java.util.Optional; -import java.util.concurrent.ExecutionException; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; public abstract class BaseEventServiceTest extends AbstractServiceTest { + long timeBeforeStartTime; + long startTime; + long eventTime; + long endTime; + long timeAfterEndTime; + + @Before + public void before() throws ParseException { + timeBeforeStartTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T11:30:00Z").getTime(); + startTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T12:00:00Z").getTime(); + eventTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T12:30:00Z").getTime(); + endTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T13:00:00Z").getTime(); + timeAfterEndTime = ISO_DATETIME_TIME_ZONE_FORMAT.parse("2016-11-01T13:30:30Z").getTime(); + } @Test public void saveEvent() throws Exception { @@ -55,18 +71,12 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { @Test public void findEventsByTypeAndTimeAscOrder() throws Exception { - long timeBeforeStartTime = LocalDateTime.of(2016, Month.NOVEMBER, 1, 11, 30).toEpochSecond(ZoneOffset.UTC); - long startTime = LocalDateTime.of(2016, Month.NOVEMBER, 1, 12, 0).toEpochSecond(ZoneOffset.UTC); - long eventTime = LocalDateTime.of(2016, Month.NOVEMBER, 1, 12, 30).toEpochSecond(ZoneOffset.UTC); - long endTime = LocalDateTime.of(2016, Month.NOVEMBER, 1, 13, 0).toEpochSecond(ZoneOffset.UTC); - long timeAfterEndTime = LocalDateTime.of(2016, Month.NOVEMBER, 1, 13, 30).toEpochSecond(ZoneOffset.UTC); - CustomerId customerId = new CustomerId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); saveEventWithProvidedTime(timeBeforeStartTime, customerId, tenantId); Event savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); - Event savedEvent2 = saveEventWithProvidedTime(eventTime+1, customerId, tenantId); - Event savedEvent3 = saveEventWithProvidedTime(eventTime+2, customerId, tenantId); + Event savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); + Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime"), startTime, endTime); @@ -86,22 +96,18 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Assert.assertTrue(events.getData().size() == 1); Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId())); Assert.assertFalse(events.hasNext()); + + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1); } @Test public void findEventsByTypeAndTimeDescOrder() throws Exception { - long timeBeforeStartTime = LocalDateTime.of(2017, Month.NOVEMBER, 1, 11, 30).toEpochSecond(ZoneOffset.UTC); - long startTime = LocalDateTime.of(2017, Month.NOVEMBER, 1, 12, 0).toEpochSecond(ZoneOffset.UTC); - long eventTime = LocalDateTime.of(2017, Month.NOVEMBER, 1, 12, 30).toEpochSecond(ZoneOffset.UTC); - long endTime = LocalDateTime.of(2017, Month.NOVEMBER, 1, 13, 0).toEpochSecond(ZoneOffset.UTC); - long timeAfterEndTime = LocalDateTime.of(2017, Month.NOVEMBER, 1, 13, 30).toEpochSecond(ZoneOffset.UTC); - CustomerId customerId = new CustomerId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); saveEventWithProvidedTime(timeBeforeStartTime, customerId, tenantId); Event savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); - Event savedEvent2 = saveEventWithProvidedTime(eventTime+1, customerId, tenantId); - Event savedEvent3 = saveEventWithProvidedTime(eventTime+2, customerId, tenantId); + Event savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); + Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); @@ -121,6 +127,8 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Assert.assertTrue(events.getData().size() == 1); Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId())); Assert.assertFalse(events.hasNext()); + + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1); } private Event saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception { diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java index aa99a88ac7..941c2d01ac 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDaoTest.java @@ -56,6 +56,8 @@ public class JpaAlarmDaoTest extends AbstractJpaDaoTest { UUID alarm3Id = UUID.fromString("d4b68f45-3e96-11e7-a884-898080180d6b"); int alarmCountBeforeSave = alarmDao.find(TenantId.fromUUID(tenantId)).size(); saveAlarm(alarm1Id, tenantId, originator1Id, "TEST_ALARM"); + //The timestamp of the startTime should be different in order for test to always work + Thread.sleep(1); saveAlarm(alarm2Id, tenantId, originator1Id, "TEST_ALARM"); saveAlarm(alarm3Id, tenantId, originator2Id, "TEST_ALARM"); int alarmCountAfterSave = alarmDao.find(TenantId.fromUUID(tenantId)).size(); diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 5d02ae84d5..3ebcf5a7ec 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -11,40 +11,53 @@ audit-log.sink.type=none cache.type=caffeine cache.maximumPoolSize=16 +cache.attributes.enabled=true #cache.type=redis -caffeine.specs.relations.timeToLiveInMinutes=1440 -caffeine.specs.relations.maxSize=100000 +cache.specs.relations.timeToLiveInMinutes=1440 +cache.specs.relations.maxSize=100000 -caffeine.specs.deviceCredentials.timeToLiveInMinutes=1440 -caffeine.specs.deviceCredentials.maxSize=100000 +cache.specs.deviceCredentials.timeToLiveInMinutes=1440 +cache.specs.deviceCredentials.maxSize=100000 -caffeine.specs.devices.timeToLiveInMinutes=1440 -caffeine.specs.devices.maxSize=100000 +cache.specs.devices.timeToLiveInMinutes=1440 +cache.specs.devices.maxSize=100000 -caffeine.specs.assets.timeToLiveInMinutes=1440 -caffeine.specs.assets.maxSize=100000 +cache.specs.sessions.timeToLiveInMinutes=1440 +cache.specs.sessions.maxSize=100000 -caffeine.specs.entityViews.timeToLiveInMinutes=1440 -caffeine.specs.entityViews.maxSize=100000 +cache.specs.assets.timeToLiveInMinutes=1440 +cache.specs.assets.maxSize=100000 -caffeine.specs.claimDevices.timeToLiveInMinutes=1440 -caffeine.specs.claimDevices.maxSize=100000 +cache.specs.entityViews.timeToLiveInMinutes=1440 +cache.specs.entityViews.maxSize=100000 -caffeine.specs.tenantProfiles.timeToLiveInMinutes=1440 -caffeine.specs.tenantProfiles.maxSize=100000 +cache.specs.claimDevices.timeToLiveInMinutes=1440 +cache.specs.claimDevices.maxSize=100000 -caffeine.specs.deviceProfiles.timeToLiveInMinutes=1440 -caffeine.specs.deviceProfiles.maxSize=100000 +cache.specs.securitySettings.timeToLiveInMinutes=1440 +cache.specs.securitySettings.maxSize=100000 -caffeine.specs.otaPackages.timeToLiveInMinutes=1440 -caffeine.specs.otaPackages.maxSize=100000 +cache.specs.tenantProfiles.timeToLiveInMinutes=1440 +cache.specs.tenantProfiles.maxSize=100000 -caffeine.specs.otaPackagesData.timeToLiveInMinutes=1440 -caffeine.specs.otaPackagesData.maxSize=100000 +cache.specs.deviceProfiles.timeToLiveInMinutes=1440 +cache.specs.deviceProfiles.maxSize=100000 -caffeine.specs.edges.timeToLiveInMinutes=1440 -caffeine.specs.edges.maxSize=100000 +cache.specs.attributes.timeToLiveInMinutes=1440 +cache.specs.attributes.maxSize=100000 + +cache.specs.tokensOutdatageTime.timeToLiveInMinutes=1440 +cache.specs.tokensOutdatageTime.maxSize=100000 + +cache.specs.otaPackages.timeToLiveInMinutes=1440 +cache.specs.otaPackages.maxSize=100000 + +cache.specs.otaPackagesData.timeToLiveInMinutes=1440 +cache.specs.otaPackagesData.maxSize=100000 + +cache.specs.edges.timeToLiveInMinutes=1440 +cache.specs.edges.maxSize=100000 redis.connection.host=localhost diff --git a/pom.xml b/pom.xml index 02297203e6..b733975383 100755 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,7 @@ 2.0.51.Final 1.7.0 4.8.0 - 2.19.1 + 3.0.0-M6 3.0.2 3.0.4 1.6.3 @@ -642,7 +642,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M5 + ${surefire.version} --illegal-access=permit diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 1c38a88709..f5752154c9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -15,7 +15,6 @@ */ package org.thingsboard.rule.engine.metadata; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.json.JsonWriteFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -24,6 +23,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonParseException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.thingsboard.rule.engine.api.TbContext; @@ -50,6 +50,7 @@ import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; +@Slf4j public abstract class TbAbstractGetAttributesNode implements TbNode { private static ObjectMapper mapper = new ObjectMapper(); @@ -112,6 +113,7 @@ public abstract class TbAbstractGetAttributesNode> attributeKvEntryListFuture = ctx.getAttributesService().find(ctx.getTenantId(), entityId, scope, keys); return Futures.transform(attributeKvEntryListFuture, attributeKvEntryList -> { + log.warn("[{}][{}][{}][{}] Lookup attribute result: {}", ctx.getTenantId(), entityId, scope, keys, attributeKvEntryList); if (!CollectionUtils.isEmpty(attributeKvEntryList)) { List existingAttributesKvEntry = attributeKvEntryList.stream().filter(attributeKvEntry -> keys.contains(attributeKvEntry.getKey())).collect(Collectors.toList()); existingAttributesKvEntry.forEach(kvEntry -> msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString()));