Browse Source

refactoring after review

pull/7984/head
ShvaykaD 4 years ago
parent
commit
2f26b8dbc0
  1. 4
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  2. 58
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  3. 90
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbRuleNodeCacheService.java
  4. 3
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java
  5. 95
      application/src/main/java/org/thingsboard/server/service/rule/DefaultRuleNodeCacheService.java
  6. 22
      application/src/main/java/org/thingsboard/server/service/rule/TbRuleNodeCacheService.java
  7. 4
      common/cache/pom.xml
  8. 49
      common/cache/src/main/java/org/thingsboard/server/cache/rule/DefaultRuleNodeCache.java
  9. 114
      common/cache/src/main/java/org/thingsboard/server/cache/rule/RedisRuleNodeCache.java
  10. 23
      common/cache/src/main/java/org/thingsboard/server/cache/rule/RuleNodeCache.java
  11. 25
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleNodeCacheService.java
  12. 22
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java
  13. 7
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java
  14. 2
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNodeException.java
  15. 21
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java
  16. 80
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbMsgDeduplicationNodeTest.java

4
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -36,6 +36,7 @@ import org.thingsboard.script.api.js.JsInvokeService;
import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.actors.service.ActorService;
import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
import org.thingsboard.server.cache.rule.RuleNodeCache;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.LifecycleEvent;
@ -96,7 +97,6 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
import org.thingsboard.server.service.rpc.TbRpcService;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.rule.RuleNodeCacheService;
import org.thingsboard.server.service.session.DeviceSessionCacheService;
import org.thingsboard.server.service.sms.SmsExecutorService;
import org.thingsboard.server.service.state.DeviceStateService;
@ -237,7 +237,7 @@ public class ActorSystemContext {
@Autowired
@Getter
private RuleNodeCacheService ruleNodeCacheService;
private RuleNodeCache ruleNodeCache;
@Autowired
private PartitionService partitionService;

58
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -31,6 +31,7 @@ import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache;
import org.thingsboard.rule.engine.api.RuleEngineRpcService;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.rule.engine.api.RuleNodeCacheService;
import org.thingsboard.rule.engine.api.ScriptEngine;
import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.rule.engine.api.TbContext;
@ -98,6 +99,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.service.rule.TbRuleNodeCacheService;
import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
import org.thingsboard.server.service.script.RuleNodeTbelScriptEngine;
@ -119,11 +121,13 @@ class DefaultTbContext implements TbContext {
private final ActorSystemContext mainCtx;
private final String ruleChainName;
private final RuleNodeCtx nodeCtx;
private final TbRuleNodeCacheService ruleNodeCacheService;
public DefaultTbContext(ActorSystemContext mainCtx, String ruleChainName, RuleNodeCtx nodeCtx) {
this.mainCtx = mainCtx;
this.ruleChainName = ruleChainName;
this.nodeCtx = nodeCtx;
this.ruleNodeCacheService = new DefaultTbRuleNodeCacheService(getSelfId(), mainCtx.getRuleNodeCache());
}
@Override
@ -734,58 +738,8 @@ class DefaultTbContext implements TbContext {
}
@Override
public void addToRuleNodeCache(String key, TbMsg msg) {
mainCtx.getRuleNodeCacheService().add(this.getSelfId(), key, msg);
}
@Override
public void addToRuleNodeCache(String key, List<TbMsg> tbMsgList) {
mainCtx.getRuleNodeCacheService().add(this.getSelfId(), key, tbMsgList);
}
@Override
public void addToRuleNodeCache(String key, byte[]... values) {
mainCtx.getRuleNodeCacheService().add(this.getSelfId(), key, values);
}
@Override
public void addToRuleNodeCache(String key, String value) {
mainCtx.getRuleNodeCacheService().add(this.getSelfId(), key, value.getBytes());
}
@Override
public Set<byte[]> getFromRuleNodeCache(String key) {
return mainCtx.getRuleNodeCacheService().get(this.getSelfId(), key);
}
@Override
public Set<TbMsg> getFromRuleNodeCache(String key, String queueName) {
return mainCtx.getRuleNodeCacheService().get(this.getSelfId(), key, queueName);
}
@Override
public void removeFromRuleNodeCache(String key, TbMsg msg) {
mainCtx.getRuleNodeCacheService().remove(this.getSelfId(), key, msg);
}
@Override
public void removeFromRuleNodeCache(String key, List<TbMsg> tbMsgList) {
mainCtx.getRuleNodeCacheService().remove(this.getSelfId(), key, tbMsgList);
}
@Override
public void removeFromRuleNodeCache(String key, byte[]... values) {
mainCtx.getRuleNodeCacheService().remove(this.getSelfId(), key, values);
}
@Override
public void removeFromRuleNodeCache(String key, String value) {
mainCtx.getRuleNodeCacheService().remove(this.getSelfId(), key, value.getBytes());
}
@Override
public void evictFromRuleNodeCache(String key) {
mainCtx.getRuleNodeCacheService().evict(this.getSelfId(), key);
public RuleNodeCacheService getRuleNodeCacheService() {
return ruleNodeCacheService;
}
@Override

90
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbRuleNodeCacheService.java

@ -0,0 +1,90 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.actors.ruleChain;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import org.thingsboard.server.cache.rule.RuleNodeCache;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.service.rule.TbRuleNodeCacheService;
import java.util.List;
import java.util.Set;
@Data
@RequiredArgsConstructor
public class DefaultTbRuleNodeCacheService implements TbRuleNodeCacheService {
private final RuleNodeId ruleNodeId;
private final RuleNodeCache cache;
@Override
public void add(String key, String value) {
cache.add(toRuleNodeCacheKey(key), value);
}
@Override
public void add(String key, EntityId id) {
cache.add(toRuleNodeCacheKey(key), id);
}
@Override
public void add(String key, TbMsg value) {
cache.add(toRuleNodeCacheKey(key), value);
}
@Override
public void removeTbMsgList(String key, List<TbMsg> values) {
cache.removeTbMsgList(toRuleNodeCacheKey(key), values);
}
@Override
public void removeStringList(String key, List<String> values) {
cache.removeStringList(toRuleNodeCacheKey(key), values);
}
@Override
public void removeEntityIdList(String key, List<EntityId> values) {
cache.removeEntityIdList(toRuleNodeCacheKey(key), values);
}
@Override
public Set<String> getStrings(String key) {
return cache.getStringSetByKey(toRuleNodeCacheKey(key));
}
@Override
public Set<EntityId> getEntityIds(String key) {
return cache.getEntityIdSetByKey(toRuleNodeCacheKey(key));
}
@Override
public Set<TbMsg> getTbMsgs(String key, String queueName) {
return cache.getTbMsgSetByKey(toRuleNodeCacheKey(key), queueName);
}
@Override
public void evict(String key) {
cache.evict(key);
}
private String toRuleNodeCacheKey(String key) {
return String.format("%s::%s", ruleNodeId.toString(), key);
}
}

3
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java

@ -88,8 +88,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor<RuleNod
@Override
public void stop(TbActorCtx context) {
if (tbNode != null) {
tbNode.destroy();
tbNode.onRemove(defaultCtx);
tbNode.destroy(defaultCtx);
state = ComponentLifecycleState.SUSPENDED;
}
}

95
application/src/main/java/org/thingsboard/server/service/rule/DefaultRuleNodeCacheService.java

@ -1,95 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.rule;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.rule.RuleNodeCache;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.queue.util.TbCoreComponent;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class DefaultRuleNodeCacheService implements RuleNodeCacheService {
private final RuleNodeCache ruleNodeCache;
@Override
public void add(RuleNodeId ruleNodeId, String key, byte[]... values) {
ruleNodeCache.add(toRuleNodeCacheKey(ruleNodeId, key), values);
}
@Override
public void add(RuleNodeId ruleNodeId, String key, TbMsg value) {
ruleNodeCache.add(toRuleNodeCacheKey(ruleNodeId, key), TbMsg.toByteArray(value));
}
@Override
public void add(RuleNodeId ruleNodeId, String key, List<TbMsg> values) {
ruleNodeCache.add(toRuleNodeCacheKey(ruleNodeId, key), tbMsgListToBytes(values));
}
@Override
public Set<byte[]> get(RuleNodeId ruleNodeId, String key) {
return ruleNodeCache.get(toRuleNodeCacheKey(ruleNodeId, key));
}
@Override
public Set<TbMsg> get(RuleNodeId ruleNodeId, String key, String queueName) {
return ruleNodeCache.get(toRuleNodeCacheKey(ruleNodeId, key))
.stream()
.map(bytes -> TbMsg.fromBytes(queueName, bytes, TbMsgCallback.EMPTY))
.collect(Collectors.toSet());
}
@Override
public void remove(RuleNodeId ruleNodeId, String key, byte[]... values) {
ruleNodeCache.remove(toRuleNodeCacheKey(ruleNodeId, key), values);
}
@Override
public void remove(RuleNodeId ruleNodeId, String key, TbMsg value) {
ruleNodeCache.remove(toRuleNodeCacheKey(ruleNodeId, key), TbMsg.toByteArray(value));
}
@Override
public void remove(RuleNodeId ruleNodeId, String key, List<TbMsg> values) {
ruleNodeCache.remove(toRuleNodeCacheKey(ruleNodeId, key), tbMsgListToBytes(values));
}
@Override
public void evict(RuleNodeId ruleNodeId, String key) {
ruleNodeCache.evict(toRuleNodeCacheKey(ruleNodeId, key));
}
private String toRuleNodeCacheKey(RuleNodeId ruleNodeId, String key) {
return String.format("%s::%s", ruleNodeId.toString(), key);
}
private byte[][] tbMsgListToBytes(List<TbMsg> values) {
return values.stream()
.map(TbMsg::toByteArray)
.toArray(byte[][]::new);
}
}

22
application/src/main/java/org/thingsboard/server/service/rule/TbRuleNodeCacheService.java

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.rule;
import org.thingsboard.rule.engine.api.RuleNodeCacheService;
public interface TbRuleNodeCacheService extends RuleNodeCacheService {
}

4
common/cache/pom.xml

@ -95,6 +95,10 @@
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>message</artifactId>
</dependency>
</dependencies>
<build>

49
common/cache/src/main/java/org/thingsboard/server/cache/rule/DefaultRuleNodeCache.java

@ -15,49 +15,60 @@
*/
package org.thingsboard.server.cache.rule;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.msg.TbMsg;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
@Service
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true)
public class DefaultRuleNodeCache implements RuleNodeCache {
private final Map<String, Set<byte[]>> cache;
@Override
public void add(String key, String value) {
}
@Override
public void add(String key, EntityId value) {
}
@Override
public void add(String key, TbMsg value) {
}
public DefaultRuleNodeCache() {
cache = new HashMap<>();
@Override
public void removeStringList(String key, List<String> values) {
}
@Override
public void removeEntityIdList(String key, List<EntityId> values) {
}
@Override
public void removeTbMsgList(String key, List<TbMsg> values) {
}
@Override
public void add(String key, byte[]... values) {
Set<byte[]> data = cache.computeIfAbsent(key, s -> new TreeSet<>(Arrays::compare));
data.addAll(Arrays.asList(values));
public Set<String> getStringSetByKey(String key) {
return Collections.emptySet();
}
@Override
public void remove(String key, byte[]... values) {
cache.computeIfPresent(key, (id, data) -> {
Arrays.asList(values).forEach(data::remove);
return data;
});
public Set<EntityId> getEntityIdSetByKey(String key) {
return Collections.emptySet();
}
@Override
public Set<byte[]> get(String key) {
return cache.getOrDefault(key, Collections.emptySet());
public Set<TbMsg> getTbMsgSetByKey(String key, String queueName) {
return Collections.emptySet();
}
@Override
public void evict(String key) {
cache.remove(key);
}
}

114
common/cache/src/main/java/org/thingsboard/server/cache/rule/RedisRuleNodeCache.java

@ -20,8 +20,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
import org.springframework.util.SerializationUtils;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@ -31,31 +38,116 @@ public class RedisRuleNodeCache implements RuleNodeCache {
private final RedisConnectionFactory redisConnectionFactory;
@Override
public void add(String key, byte[]... values) {
public void add(String key, String value) {
processAdd(key, value.getBytes());
}
@Override
public void add(String key, EntityId value) {
processAdd(key, SerializationUtils.serialize(value));
}
@Override
public void add(String key, TbMsg value) {
processAdd(key, TbMsg.toByteArray(value));
}
@Override
public void removeStringList(String key, List<String> values) {
processRemove(key, stringListToBytes(values));
}
@Override
public void removeEntityIdList(String key, List<EntityId> values) {
processRemove(key, entityIdListToBytes(values));
}
@Override
public void removeTbMsgList(String key, List<TbMsg> values) {
processRemove(key, tbMsgListToBytes(values));
}
@Override
public Set<String> getStringSetByKey(String key) {
return toStringSet(processGetMembers(key));
}
@Override
public Set<EntityId> getEntityIdSetByKey(String key) {
return toEntityIdSet(processGetMembers(key));
}
@Override
public Set<TbMsg> getTbMsgSetByKey(String key, String queueName) {
return toTbMsgSet(processGetMembers(key), queueName);
}
@Override
public void evict(String key) {
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
connection.setCommands().sAdd(key.getBytes(), values);
connection.del(key.getBytes());
}
}
@Override
public void remove(String key, byte[]... values) {
private void processAdd(String key, byte[] value) {
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
connection.setCommands().sRem(key.getBytes(), values);
connection.setCommands().sAdd(key.getBytes(), value);
}
}
@Override
public Set<byte[]> get(String key) {
private void processRemove(String key, byte[][] values) {
if (values.length == 0) {
return;
}
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
return connection.setCommands().sMembers(key.getBytes());
connection.setCommands().sRem(key.getBytes(), values);
}
}
@Override
public void evict(String key) {
private Set<byte[]> processGetMembers(String key) {
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
connection.del(key.getBytes());
Set<byte[]> bytes = connection.setCommands().sMembers(key.getBytes());
if (bytes == null) {
return Collections.emptySet();
}
return bytes;
}
}
private byte[][] stringListToBytes(List<String> values) {
return values.stream()
.map(String::getBytes)
.toArray(byte[][]::new);
}
private byte[][] entityIdListToBytes(List<EntityId> values) {
return values.stream()
.map(SerializationUtils::serialize)
.toArray(byte[][]::new);
}
private byte[][] tbMsgListToBytes(List<TbMsg> values) {
return values.stream()
.map(TbMsg::toByteArray)
.toArray(byte[][]::new);
}
private Set<String> toStringSet(Set<byte[]> values) {
return values.stream()
.map(String::new)
.collect(Collectors.toSet());
}
private Set<EntityId> toEntityIdSet(Set<byte[]> values) {
return values.stream()
.map(bytes -> (EntityId) SerializationUtils.deserialize(bytes))
.collect(Collectors.toSet());
}
private Set<TbMsg> toTbMsgSet(Set<byte[]> values, String queueName) {
return values.stream()
.map(bytes -> TbMsg.fromBytes(queueName, bytes, TbMsgCallback.EMPTY))
.collect(Collectors.toSet());
}
}

23
common/cache/src/main/java/org/thingsboard/server/cache/rule/RuleNodeCache.java

@ -15,16 +15,31 @@
*/
package org.thingsboard.server.cache.rule;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.msg.TbMsg;
import java.util.List;
import java.util.Set;
public interface RuleNodeCache {
void add(String key, byte[]... values);
void add(String key, String value);
void remove(String key, byte[]... values);
void add(String key, EntityId value);
Set<byte[]> get(String key);
void add(String key, TbMsg value);
void evict(String key);
void removeStringList(String key, List<String> values);
void removeEntityIdList(String key, List<EntityId> values);
void removeTbMsgList(String key, List<TbMsg> values);
Set<String> getStringSetByKey(String key);
Set<EntityId> getEntityIdSetByKey(String key);
Set<TbMsg> getTbMsgSetByKey(String key, String queueName);
void evict(String key);
}

25
application/src/main/java/org/thingsboard/server/service/rule/RuleNodeCacheService.java → rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleNodeCacheService.java

@ -13,9 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.rule;
package org.thingsboard.rule.engine.api;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.msg.TbMsg;
import java.util.List;
@ -23,21 +23,24 @@ import java.util.Set;
public interface RuleNodeCacheService {
void add(RuleNodeId ruleNodeId, String key, byte[]... values);
void add(String key, String value);
void add(RuleNodeId ruleNodeId, String key, TbMsg value);
void add(String key, EntityId id);
void add(RuleNodeId ruleNodeId, String key, List<TbMsg> values);
void add(String key, TbMsg value);
Set<byte[]> get(RuleNodeId ruleNodeId, String key);
void removeStringList(String key, List<String> values);
Set<TbMsg> get(RuleNodeId ruleNodeId, String key, String queueName);
void removeEntityIdList(String key, List<EntityId> values);
void remove(RuleNodeId ruleNodeId, String key, byte[]... values);
void removeTbMsgList(String key, List<TbMsg> values);
void remove(RuleNodeId ruleNodeId, String key, TbMsg value);
Set<String> getStrings(String key);
void remove(RuleNodeId ruleNodeId, String key, List<TbMsg> values);
Set<EntityId> getEntityIds(String key);
Set<TbMsg> getTbMsgs(String key, String queueName);
void evict(String key);
void evict(RuleNodeId ruleNodeId, String key);
}

22
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java

@ -317,27 +317,7 @@ public interface TbContext {
void clearRuleNodeStates();
void addToRuleNodeCache(String key, TbMsg msg);
void addToRuleNodeCache(String key, List<TbMsg> tbMsgList);
void addToRuleNodeCache(String key, String value);
void addToRuleNodeCache(String key, byte[]... values);
Set<byte[]> getFromRuleNodeCache(String key);
Set<TbMsg> getFromRuleNodeCache(String key, String queueName);
void removeFromRuleNodeCache(String key, TbMsg msg);
void removeFromRuleNodeCache(String key, List<TbMsg> tbMsgList);
void removeFromRuleNodeCache(String key, byte[]... values);
void removeFromRuleNodeCache(String key, String value);
void evictFromRuleNodeCache(String key);
RuleNodeCacheService getRuleNodeCacheService();
void addTenantProfileListener(Consumer<TenantProfile> listener);

7
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNode.java

@ -31,8 +31,11 @@ public interface TbNode {
default void destroy() {}
default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {}
// todo: add reason for destroy
default void destroy(TbContext ctx) {
destroy();
}
default void onRemove(TbContext ctx) {}
default void onPartitionChangeMsg(TbContext ctx, PartitionChangeMsg msg) {}
}

2
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbNodeException.java

@ -15,8 +15,6 @@
*/
package org.thingsboard.rule.engine.api;
import com.fasterxml.jackson.core.JsonProcessingException;
/**
* Created by ashvayka on 19.01.18.
*/

21
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java

@ -18,7 +18,6 @@ package org.thingsboard.rule.engine.deduplication;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.SerializationUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.RuleNode;
import org.thingsboard.rule.engine.api.TbContext;
@ -43,7 +42,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@RuleNode(
type = ComponentType.ACTION,
@ -97,14 +95,14 @@ public class TbMsgDeduplicationNode implements TbNode {
}
@Override
public void onRemove(TbContext ctx) {
public void destroy(TbContext ctx) {
deduplicationMap.clear();
Set<EntityId> deduplicationIds = getDeduplicationIds(ctx);
if (deduplicationIds == null) {
return;
}
deduplicationIds.forEach(id -> ctx.evictFromRuleNodeCache(id.toString()));
ctx.evictFromRuleNodeCache(DEDUPLICATION_IDS_CACHE_KEY);
deduplicationIds.forEach(id -> ctx.getRuleNodeCacheService().evict(id.toString()));
ctx.getRuleNodeCacheService().evict(DEDUPLICATION_IDS_CACHE_KEY);
}
private void getDeduplicationDataFromCacheAndSchedule(TbContext ctx) {
@ -113,7 +111,7 @@ public class TbMsgDeduplicationNode implements TbNode {
return;
}
deduplicationIds.forEach(id -> {
List<TbMsg> tbMsgs = new ArrayList<>(ctx.getFromRuleNodeCache(id.toString(), config.getQueueName()));
List<TbMsg> tbMsgs = new ArrayList<>(ctx.getRuleNodeCacheService().getTbMsgs(id.toString(), config.getQueueName()));
DeduplicationData deduplicationData = new DeduplicationData();
deduplicationData.addAll(tbMsgs);
deduplicationMap.put(id, deduplicationData);
@ -122,10 +120,7 @@ public class TbMsgDeduplicationNode implements TbNode {
}
private Set<EntityId> getDeduplicationIds(TbContext ctx) {
Set<EntityId> deduplicationIds = ctx.getFromRuleNodeCache(TbMsgDeduplicationNode.DEDUPLICATION_IDS_CACHE_KEY)
.stream()
.map(bytes -> (EntityId) SerializationUtils.deserialize(bytes))
.collect(Collectors.toSet());
Set<EntityId> deduplicationIds = ctx.getRuleNodeCacheService().getEntityIds(TbMsgDeduplicationNode.DEDUPLICATION_IDS_CACHE_KEY);
if (deduplicationIds.isEmpty()) {
return null;
}
@ -138,10 +133,10 @@ public class TbMsgDeduplicationNode implements TbNode {
if (deduplicationMsgs.size() < config.getMaxPendingMsgs()) {
log.trace("[{}][{}] Adding msg: [{}][{}] to the pending msgs map ...", ctx.getSelfId(), id, msg.getId(), msg.getMetaDataTs());
if (deduplicationMsgs.isEmpty()) {
ctx.addToRuleNodeCache(DEDUPLICATION_IDS_CACHE_KEY, SerializationUtils.serialize(id));
ctx.getRuleNodeCacheService().add(DEDUPLICATION_IDS_CACHE_KEY, id);
}
deduplicationMsgs.add(msg);
ctx.addToRuleNodeCache(id.toString(), msg);
ctx.getRuleNodeCacheService().add(id.toString(), msg);
ctx.ack(msg);
scheduleTickMsg(ctx, id, deduplicationMsgs);
} else {
@ -252,7 +247,7 @@ public class TbMsgDeduplicationNode implements TbNode {
ctx.enqueueForTellNext(outMsg, TbRelationTypes.SUCCESS,
() -> {
log.trace("[{}][{}][{}] Successfully enqueue deduplication result message!", ctx.getSelfId(), outMsg.getOriginator(), retryAttempt);
ctx.removeFromRuleNodeCache(outMsg.getOriginator().toString(), msgsToRemoveFromCache);
ctx.getRuleNodeCacheService().removeTbMsgList(outMsg.getOriginator().toString(), msgsToRemoveFromCache);
},
throwable -> {
log.trace("[{}][{}][{}] Failed to enqueue deduplication output message due to: ", ctx.getSelfId(), outMsg.getOriginator(), retryAttempt, throwable);

80
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbMsgDeduplicationNodeTest.java

@ -28,6 +28,7 @@ import org.mockito.ArgumentMatchers;
import org.mockito.stubbing.Answer;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.rule.engine.api.RuleNodeCacheService;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
@ -44,6 +45,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.session.SessionMsgType;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.UUID;
@ -58,11 +60,15 @@ import java.util.function.Consumer;
import java.util.stream.Collectors;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -74,6 +80,7 @@ public class TbMsgDeduplicationNodeTest {
private static final String MAIN_QUEUE_NAME = "Main";
private static final String HIGH_PRIORITY_QUEUE_NAME = "HighPriority";
private static final String TB_MSG_DEDUPLICATION_TIMEOUT_MSG = "TbMsgDeduplicationNodeMsg";
private static final String DEDUPLICATION_IDS_CACHE_KEY = "deduplication_ids";
private TbContext ctx;
@ -86,18 +93,24 @@ public class TbMsgDeduplicationNodeTest {
private TbMsgDeduplicationNode node;
private TbMsgDeduplicationNodeConfiguration config;
private TbNodeConfiguration nodeConfiguration;
private RuleNodeCacheService ruleNodeCacheService;
private CountDownLatch awaitTellSelfLatch;
@BeforeEach
public void init() throws TbNodeException {
ctx = mock(TbContext.class);
ruleNodeCacheService = mock(RuleNodeCacheService.class);
tenantId = TenantId.fromUUID(UUID.randomUUID());
RuleNodeId ruleNodeId = new RuleNodeId(UUID.randomUUID());
when(ctx.getSelfId()).thenReturn(ruleNodeId);
when(ctx.getTenantId()).thenReturn(tenantId);
when(ctx.getRuleNodeCacheService()).thenReturn(ruleNodeCacheService);
when(ruleNodeCacheService.getStrings(anyString())).thenReturn(Collections.emptySet());
when(ruleNodeCacheService.getEntityIds(anyString())).thenReturn(Collections.emptySet());
when(ruleNodeCacheService.getTbMsgs(anyString(), anyString())).thenReturn(Collections.emptySet());
doAnswer((Answer<TbMsg>) invocationOnMock -> {
String type = (String) (invocationOnMock.getArguments())[1];
@ -173,10 +186,22 @@ public class TbMsgDeduplicationNodeTest {
ArgumentCaptor<Runnable> successCaptor = ArgumentCaptor.forClass(Runnable.class);
ArgumentCaptor<Consumer<Throwable>> failureCaptor = ArgumentCaptor.forClass(Consumer.class);
verify(ruleNodeCacheService, times(1)).getEntityIds(anyString());
verify(ruleNodeCacheService, times(1)).add(eq(DEDUPLICATION_IDS_CACHE_KEY), eq(deviceId));
verify(ruleNodeCacheService, times(msgCount)).add(eq(deviceId.toString()), any(TbMsg.class));
verify(node, times(msgCount + wantedNumberOfTellSelfInvocation + 1)).onMsg(eq(ctx), any());
verify(ctx, times(msgCount)).ack(any());
verify(ctx, times(1)).tellFailure(eq(msgToReject), any());
verify(node, times(msgCount + wantedNumberOfTellSelfInvocation + 1)).onMsg(eq(ctx), any());
verify(ctx, times(1)).enqueueForTellNext(newMsgCaptor.capture(), eq(TbRelationTypes.SUCCESS), successCaptor.capture(), failureCaptor.capture());
for (Runnable valueCaptor : successCaptor.getAllValues()) {
valueCaptor.run();
}
verify(ruleNodeCacheService, times(1)).removeTbMsgList(eq(deviceId.toString()), anyList());
verify(ctx, never()).schedule(any(), anyLong(), any());
Assertions.assertEquals(TbMsg.toByteString(inputMsgs.get(0)), TbMsg.toByteString(newMsgCaptor.getValue()));
}
@ -212,10 +237,22 @@ public class TbMsgDeduplicationNodeTest {
ArgumentCaptor<Runnable> successCaptor = ArgumentCaptor.forClass(Runnable.class);
ArgumentCaptor<Consumer<Throwable>> failureCaptor = ArgumentCaptor.forClass(Consumer.class);
verify(ruleNodeCacheService, times(1)).getEntityIds(anyString());
verify(ruleNodeCacheService, times(1)).add(eq(DEDUPLICATION_IDS_CACHE_KEY), eq(deviceId));
verify(ruleNodeCacheService, times(msgCount)).add(eq(deviceId.toString()), any(TbMsg.class));
verify(node, times(msgCount + wantedNumberOfTellSelfInvocation + 1)).onMsg(eq(ctx), any());
verify(ctx, times(msgCount)).ack(any());
verify(ctx, times(1)).tellFailure(eq(msgToReject), any());
verify(node, times(msgCount + wantedNumberOfTellSelfInvocation + 1)).onMsg(eq(ctx), any());
verify(ctx, times(1)).enqueueForTellNext(newMsgCaptor.capture(), eq(TbRelationTypes.SUCCESS), successCaptor.capture(), failureCaptor.capture());
for (Runnable valueCaptor : successCaptor.getAllValues()) {
valueCaptor.run();
}
verify(ruleNodeCacheService, times(1)).removeTbMsgList(eq(deviceId.toString()), anyList());
verify(ctx, never()).schedule(any(), anyLong(), any());
Assertions.assertEquals(TbMsg.toByteString(msgWithLatestTs), TbMsg.toByteString(newMsgCaptor.getValue()));
}
@ -247,10 +284,21 @@ public class TbMsgDeduplicationNodeTest {
ArgumentCaptor<Runnable> successCaptor = ArgumentCaptor.forClass(Runnable.class);
ArgumentCaptor<Consumer<Throwable>> failureCaptor = ArgumentCaptor.forClass(Consumer.class);
verify(ctx, times(msgCount)).ack(any());
verify(ruleNodeCacheService, times(1)).getEntityIds(anyString());
verify(ruleNodeCacheService, times(1)).add(eq(DEDUPLICATION_IDS_CACHE_KEY), eq(deviceId));
verify(ruleNodeCacheService, times(msgCount)).add(eq(deviceId.toString()), any(TbMsg.class));
verify(node, times(msgCount + wantedNumberOfTellSelfInvocation)).onMsg(eq(ctx), any());
verify(ctx, times(msgCount)).ack(any());
verify(ctx, times(1)).enqueueForTellNext(newMsgCaptor.capture(), eq(TbRelationTypes.SUCCESS), successCaptor.capture(), failureCaptor.capture());
for (Runnable valueCaptor : successCaptor.getAllValues()) {
valueCaptor.run();
}
verify(ruleNodeCacheService, times(1)).removeTbMsgList(eq(deviceId.toString()), anyList());
verify(ctx, never()).schedule(any(), anyLong(), any());
Assertions.assertEquals(1, newMsgCaptor.getAllValues().size());
TbMsg outMessage = newMsgCaptor.getAllValues().get(0);
Assertions.assertEquals(getMergedData(inputMsgs), outMessage.getData());
@ -293,10 +341,21 @@ public class TbMsgDeduplicationNodeTest {
ArgumentCaptor<Runnable> successCaptor = ArgumentCaptor.forClass(Runnable.class);
ArgumentCaptor<Consumer<Throwable>> failureCaptor = ArgumentCaptor.forClass(Consumer.class);
verify(ctx, times(msgCount)).ack(any());
verify(ruleNodeCacheService, times(1)).getEntityIds(anyString());
verify(ruleNodeCacheService, times(1)).add(eq(DEDUPLICATION_IDS_CACHE_KEY), eq(deviceId));
verify(ruleNodeCacheService, times(msgCount)).add(eq(deviceId.toString()), any(TbMsg.class));
verify(node, times(msgCount + wantedNumberOfTellSelfInvocation)).onMsg(eq(ctx), any());
verify(ctx, times(msgCount)).ack(any());
verify(ctx, times(2)).enqueueForTellNext(newMsgCaptor.capture(), eq(TbRelationTypes.SUCCESS), successCaptor.capture(), failureCaptor.capture());
for (Runnable valueCaptor : successCaptor.getAllValues()) {
valueCaptor.run();
}
verify(ruleNodeCacheService, times(2)).removeTbMsgList(eq(deviceId.toString()), anyList());
verify(ctx, never()).schedule(any(), anyLong(), any());
List<TbMsg> resultMsgs = newMsgCaptor.getAllValues();
Assertions.assertEquals(2, resultMsgs.size());
@ -347,10 +406,21 @@ public class TbMsgDeduplicationNodeTest {
ArgumentCaptor<Runnable> successCaptor = ArgumentCaptor.forClass(Runnable.class);
ArgumentCaptor<Consumer<Throwable>> failureCaptor = ArgumentCaptor.forClass(Consumer.class);
verify(ctx, times(msgCount)).ack(any());
verify(ruleNodeCacheService, times(1)).getEntityIds(anyString());
verify(ruleNodeCacheService, times(1)).add(eq(DEDUPLICATION_IDS_CACHE_KEY), eq(deviceId));
verify(ruleNodeCacheService, times(msgCount)).add(eq(deviceId.toString()), any(TbMsg.class));
verify(node, times(msgCount + wantedNumberOfTellSelfInvocation)).onMsg(eq(ctx), any());
verify(ctx, times(msgCount)).ack(any());
verify(ctx, times(2)).enqueueForTellNext(newMsgCaptor.capture(), eq(TbRelationTypes.SUCCESS), successCaptor.capture(), failureCaptor.capture());
for (Runnable valueCaptor : successCaptor.getAllValues()) {
valueCaptor.run();
}
verify(ruleNodeCacheService, times(2)).removeTbMsgList(eq(deviceId.toString()), anyList());
verify(ctx, never()).schedule(any(), anyLong(), any());
List<TbMsg> resultMsgs = newMsgCaptor.getAllValues();
Assertions.assertEquals(2, resultMsgs.size());
List<ByteString> resultMsgsByteStrings = resultMsgs.stream().map(TbMsg::toByteString).collect(Collectors.toList());

Loading…
Cancel
Save