From f9b7f6540ee33e31f5182e1e3e68efb93f588d0f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 27 Sep 2022 14:57:18 +0300 Subject: [PATCH] Math Function rule node --- pom.xml | 6 + rule-engine/rule-engine-components/pom.xml | 6 + .../rule/engine/math/TbMathArgument.java | 5 +- .../rule/engine/math/TbMathNode.java | 29 ++++- .../engine/math/TbMathNodeConfiguration.java | 1 + .../math/TbRuleNodeMathFunctionType.java | 4 +- .../src/test/java/math/TbMathNodeTest.java | 118 +++++++++++++++++- 7 files changed, 159 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 347874faef..0a00425ef8 100755 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,7 @@ 1.12 3.0.0 6.1.0.202203080745-r + 0.4.8 1.0.0 @@ -1911,6 +1912,11 @@ org.eclipse.jgit.ssh.apache ${jgit.version} + + net.objecthunter + exp4j + ${exp4j.version} + diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 439ed41859..9488bdb031 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -121,6 +121,10 @@ javax.mail provided + + net.objecthunter + exp4j + org.springframework.boot spring-boot-starter-test @@ -139,10 +143,12 @@ org.mock-server mockserver-netty + test org.mock-server mockserver-client-java + test diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathArgument.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathArgument.java index 3c9715baae..4fb2c66baa 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathArgument.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathArgument.java @@ -24,13 +24,14 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class TbMathArgument { + private String name; private TbMathArgumentType type; private String key; private String attributeScope; private Double defaultValue; public TbMathArgument(TbMathArgumentType type, String key) { - this.type = type; - this.key = key; + this(key, type, key, null, null); } + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java index 7e97ee0510..1a8ced205c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java @@ -20,6 +20,8 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; @@ -57,6 +59,7 @@ import java.util.stream.Collectors; configClazz = TbMathNodeConfiguration.class, nodeDescription = "Apply math function and save the result into the message and/or database", nodeDetails = "Supports math operations like: ADD, SUB, MULT, DIV, etc and functions: SIN, COS, TAN, SEC, etc. " + + "Use 'CUSTOM' operation to specify complex math expressions." + "

" + "You may use constant, message field, metadata field, attribute, and latest time-series as an arguments values. " + "The result of the function may be also stored to message field, metadata field, attribute or time-series value." + @@ -65,15 +68,17 @@ import java.util.stream.Collectors; "For example, you may increase `totalWaterConsumption` based on the `deltaWaterConsumption` reported by device." + "

" + "Alternative use case is the replacement of simple JS `script` nodes with more light-weight and performant implementation. " + - "For example, you may transform Fahrenheit to Celsius (C = (F - 32) / 1.8) using combination of two math node functions: SUB 32 and DIV 1.8." + + "For example, you may transform Fahrenheit to Celsius (C = (F - 32) / 1.8) using CUSTOM operation and expression: (x - 32) / 1.8)." + "

" + "The execution is synchronized in scope of message originator (e.g. device) and server node. " + "If you have rule nodes in different rule chains, they will process messages from the same originator synchronously in the scope of the server node.", icon = "functions" + ) public class TbMathNode implements TbNode { private static final ConcurrentMap semaphores = new ConcurrentReferenceHashMap<>(); + private final ThreadLocal customExpression = new ThreadLocal<>(); private TbMathNodeConfiguration config; private boolean msgBodyToJsonConversionRequired; @@ -86,6 +91,13 @@ public class TbMathNode implements TbNode { if (argsCount < operation.getMinArgs() || argsCount > operation.getMaxArgs()) { throw new RuntimeException("Args count: " + argsCount + " does not match operation: " + operation.name()); } + if (TbRuleNodeMathFunctionType.CUSTOM.equals(operation)) { + if (StringUtils.isBlank(config.getCustomFunction())) { + throw new RuntimeException("Custom function is blank!"); + } else if (config.getCustomFunction().length() > 256) { + throw new RuntimeException("Custom function is too complex (length > 256)!"); + } + } msgBodyToJsonConversionRequired = config.getArguments().stream().anyMatch(arg -> TbMathArgumentType.MESSAGE_BODY.equals(arg.getType())); msgBodyToJsonConversionRequired = msgBodyToJsonConversionRequired || TbMathArgumentType.MESSAGE_BODY.equals(config.getResult().getType()); } @@ -304,6 +316,19 @@ public class TbMathNode implements TbNode { return apply(args.get(0), Math::toRadians); case DEG: return apply(args.get(0), Math::toDegrees); + case CUSTOM: + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(config.getCustomFunction()) + .implicitMultiplication(true) + .variables(config.getArguments().stream().map(TbMathArgument::getName).collect(Collectors.toSet())) + .build(); + customExpression.set(expr); + } + for (int i = 0; i < config.getArguments().size(); i++) { + expr.setVariable(config.getArguments().get(i).getName(), args.get(i).getValue()); + } + return expr.evaluate(); default: throw new RuntimeException("Not supported operation: " + config.getOperation()); } @@ -329,7 +354,7 @@ public class TbMathNode implements TbNode { String scope = getAttributeScope(arg.getAttributeScope()); return Futures.transform(ctx.getAttributesService().find(ctx.getTenantId(), msg.getOriginator(), scope, arg.getKey()), opt -> getTbMathArgumentValue(arg, opt, "Attribute: " + arg.getKey() + " with scope: " + scope + " not found for entity: " + msg.getOriginator()) - ,MoreExecutors.directExecutor()); + , MoreExecutors.directExecutor()); case TIME_SERIES: return Futures.transform(ctx.getTimeseriesService().findLatest(ctx.getTenantId(), msg.getOriginator(), arg.getKey()), opt -> getTbMathArgumentValue(arg, opt, "Time-series: " + arg.getKey() + " not found for entity: " + msg.getOriginator()) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNodeConfiguration.java index c706456455..3be32a9bdf 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNodeConfiguration.java @@ -26,6 +26,7 @@ public class TbMathNodeConfiguration implements NodeConfiguration arguments; + private String customFunction; private TbMathResult result; @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbRuleNodeMathFunctionType.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbRuleNodeMathFunctionType.java index 0ef95389fe..ffcd518f24 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbRuleNodeMathFunctionType.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbRuleNodeMathFunctionType.java @@ -23,7 +23,9 @@ public enum TbRuleNodeMathFunctionType { SIN, SINH, COS, COSH, TAN, TANH, ACOS, ASIN, ATAN, ATAN2(2), EXP, EXPM1, SQRT, CBRT, GET_EXP(1, 1, true), HYPOT(2), LOG, LOG10, LOG1P, CEIL(1, 1, true), FLOOR(1, 1, true), FLOOR_DIV(2), FLOOR_MOD(2), - ABS, MIN(2), MAX(2), POW, SIGNUM, RAD, DEG; + ABS, MIN(2), MAX(2), POW, SIGNUM, RAD, DEG, + + CUSTOM(0, 16, false); //Custom function based on exp4j @Getter private final int minArgs; diff --git a/rule-engine/rule-engine-components/src/test/java/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/math/TbMathNodeTest.java index aa7cbb8e50..5a0cf6eb91 100644 --- a/rule-engine/rule-engine-components/src/test/java/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/math/TbMathNodeTest.java @@ -33,8 +33,8 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.math.TbMathArgument; import org.thingsboard.rule.engine.math.TbMathArgumentType; -import org.thingsboard.rule.engine.math.TbMathNodeConfiguration; import org.thingsboard.rule.engine.math.TbMathNode; +import org.thingsboard.rule.engine.math.TbMathNodeConfiguration; import org.thingsboard.rule.engine.math.TbMathResult; import org.thingsboard.rule.engine.math.TbRuleNodeMathFunctionType; import org.thingsboard.server.common.data.DataConstants; @@ -79,6 +79,15 @@ public class TbMathNodeTest { } }; dbExecutor.init(); + initMocks(); + } + + @After + public void after() { + dbExecutor.destroy(); + } + + private void initMocks() { Mockito.reset(ctx); Mockito.reset(attributesService); Mockito.reset(tsService); @@ -88,16 +97,21 @@ public class TbMathNodeTest { lenient().when(ctx.getDbCallbackExecutor()).thenReturn(dbExecutor); } - @After - public void after() { - dbExecutor.destroy(); + private TbMathNode initNode(TbRuleNodeMathFunctionType operation, TbMathResult result, TbMathArgument... arguments) { + return initNode(operation, null, result, arguments); } + private TbMathNode initNodeWithCustomFunction(String expression, TbMathResult result, TbMathArgument... arguments) { + return initNode(TbRuleNodeMathFunctionType.CUSTOM, expression, result, arguments); + } - private TbMathNode initNode(TbRuleNodeMathFunctionType operation, TbMathResult result, TbMathArgument... arguments) { + private TbMathNode initNode(TbRuleNodeMathFunctionType operation, String expression, TbMathResult result, TbMathArgument... arguments) { try { TbMathNodeConfiguration configuration = new TbMathNodeConfiguration(); configuration.setOperation(operation); + if (TbRuleNodeMathFunctionType.CUSTOM.equals(operation)) { + configuration.setCustomFunction(expression); + } configuration.setResult(result); configuration.setArguments(Arrays.asList(arguments)); TbMathNode node = new TbMathNode(); @@ -108,6 +122,100 @@ public class TbMathNodeTest { } } + @Test + public void testExp4j() { + var node = initNodeWithCustomFunction("2a+3b", + new TbMathResult(TbMathArgumentType.MESSAGE_BODY, "result", 2, false, false, null), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "a"), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "b") + ); + + TbMsg msg = TbMsg.newMsg("TEST", originator, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()); + + node.onMsg(ctx, msg); + + ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); + Mockito.verify(ctx, Mockito.timeout(5000)).tellSuccess(msgCaptor.capture()); + + TbMsg resultMsg = msgCaptor.getValue(); + Assert.assertNotNull(resultMsg); + Assert.assertNotNull(resultMsg.getData()); + var resultJson = JacksonUtil.toJsonNode(resultMsg.getData()); + Assert.assertTrue(resultJson.has("result")); + Assert.assertEquals(10, resultJson.get("result").asInt()); + } + + @Test + public void testSimpleFunctions() { + testSimpleTwoArgumentFunction(TbRuleNodeMathFunctionType.ADD, 2.1, 2.2, 4.3); + testSimpleTwoArgumentFunction(TbRuleNodeMathFunctionType.SUB, 2.1, 2.2, -0.1); + testSimpleTwoArgumentFunction(TbRuleNodeMathFunctionType.MULT, 2.1, 2.0, 4.2); + testSimpleTwoArgumentFunction(TbRuleNodeMathFunctionType.DIV, 4.2, 2.0, 2.1); + + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.SIN, Math.toRadians(30), 0.5); + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.SIN, Math.toRadians(90), 1.0); + + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.SINH, Math.toRadians(0), 0.0); + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.COSH, Math.toRadians(0), 1.0); + + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.COS, Math.toRadians(60), 0.5); + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.COS, Math.toRadians(0), 1.0); + + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.TAN, Math.toRadians(45), 1); + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.TAN, Math.toRadians(0), 0); + + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.ABS, -1, 1); + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.SQRT, 4, 2); + testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType.CBRT, 8, 2); + } + + private void testSimpleTwoArgumentFunction(TbRuleNodeMathFunctionType function, double arg1, double arg2, double result) { + initMocks(); + + var node = initNode(function, + new TbMathResult(TbMathArgumentType.MESSAGE_BODY, "result", 2, false, false, null), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "a"), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "b") + ); + + TbMsg msg = TbMsg.newMsg("TEST", originator, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", arg1).put("b", arg2).toString()); + + node.onMsg(ctx, msg); + + ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); + Mockito.verify(ctx, Mockito.timeout(5000).times(1)).tellSuccess(msgCaptor.capture()); + + TbMsg resultMsg = msgCaptor.getValue(); + Assert.assertNotNull(resultMsg); + Assert.assertNotNull(resultMsg.getData()); + var resultJson = JacksonUtil.toJsonNode(resultMsg.getData()); + Assert.assertTrue(resultJson.has("result")); + Assert.assertEquals(result, resultJson.get("result").asDouble(), 0d); + } + + private void testSimpleOneArgumentFunction(TbRuleNodeMathFunctionType function, double arg1, double result) { + initMocks(); + + var node = initNode(function, + new TbMathResult(TbMathArgumentType.MESSAGE_BODY, "result", 2, false, false, null), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "a") + ); + + TbMsg msg = TbMsg.newMsg("TEST", originator, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", arg1).toString()); + + node.onMsg(ctx, msg); + + ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); + Mockito.verify(ctx, Mockito.timeout(5000)).tellSuccess(msgCaptor.capture()); + + TbMsg resultMsg = msgCaptor.getValue(); + Assert.assertNotNull(resultMsg); + Assert.assertNotNull(resultMsg.getData()); + var resultJson = JacksonUtil.toJsonNode(resultMsg.getData()); + Assert.assertTrue(resultJson.has("result")); + Assert.assertEquals(result, resultJson.get("result").asDouble(), 0d); + } + @Test public void test_2_plus_2_body() { var node = initNode(TbRuleNodeMathFunctionType.ADD,