From 03d94c77d7cf5ef618f17ad4e0fd72cf67f9bb85 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Thu, 20 Apr 2023 10:06:55 +0300 Subject: [PATCH 1/5] Added new function to TBEL helper --- .../thingsboard/script/api/tbel/TbUtils.java | 64 +++++++++++++++++ .../script/api/tbel/TbUtilsTest.java | 70 +++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index fa75709d83..c4f86f80a3 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -18,7 +18,9 @@ package org.thingsboard.script.api.tbel; import org.mvel2.ExecutionContext; import org.mvel2.ParserConfiguration; import org.mvel2.execution.ExecutionArrayList; +import org.mvel2.execution.ExecutionHashMap; import org.mvel2.util.MethodStub; +import org.thingsboard.server.common.data.StringUtils; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -27,8 +29,13 @@ import java.math.RoundingMode; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; public class TbUtils { @@ -89,6 +96,14 @@ public class TbUtils { byte[].class))); parserConfig.addImport("bytesToHex", new MethodStub(TbUtils.class.getMethod("bytesToHex", ExecutionArrayList.class))); + parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", + Object.class, HashMap.class))); + parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", + Object.class, HashMap.class, boolean.class))); + parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", + Object.class, HashMap.class, List.class))); + parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", + Object.class, HashMap.class, List.class, boolean.class))); } public static String btoa(String input) { @@ -316,4 +331,53 @@ public class TbUtils { } return value; } + + public static void toFlatMap(Object json, HashMap map) { + toFlatMap(json, map, new ArrayList<>(), true); + } + + public static void toFlatMap(Object json, HashMap map, boolean pathInKey) { + toFlatMap(json, map, new ArrayList<>(), pathInKey); + } + + public static void toFlatMap(Object json, HashMap map, List excludeList) { + toFlatMap(json, map, excludeList, true); + } + + public static void toFlatMap(Object json, HashMap map, List excludeList, boolean pathInKey) { + parseRecursive(json, map, excludeList, "", pathInKey); + } + + private static void parseRecursive(Object json, HashMap map, List excludeList, String path, boolean pathInKey) { + if (json instanceof Map.Entry) { + Map.Entry entry = (Map.Entry) json; + if (StringUtils.isNotBlank(path)) { + path += "."; + } + if (excludeList.contains(entry.getKey())) { + return; + } + path += entry.getKey(); + json = entry.getValue(); + } + if (json instanceof Set || json instanceof List) { + String arrayPath = path + "."; + Object[] collection = ((Collection) json).toArray(); + for (int index = 0; index < collection.length; index++) { + parseRecursive(collection[index], map, excludeList, arrayPath + index, pathInKey); + } + } else if (json instanceof Map) { + Map node = (Map) json; + for (Map.Entry entry : node.entrySet()) { + parseRecursive(entry, map, excludeList, path, pathInKey); + } + } else { + if (pathInKey) { + map.put(path, json); + } else { + String key = path.contains(".") ? path.substring(path.lastIndexOf('.') + 1) : path; + map.put(key, json); + } + } + } } diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index f4cd610b72..af10b7b49b 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -20,7 +20,9 @@ import org.junit.Test; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class TbUtilsTest { @@ -87,6 +89,74 @@ public class TbUtilsTest { Assert.assertEquals(expected, TbUtils.parseBytesToInt(data, 0, 3, false)); } + @Test + public void toFlatMap() { + HashMap inputMap = new HashMap<>(); + inputMap.put("name", "Alice"); + inputMap.put("age", 30); + inputMap.put("devices", List.of( + new HashMap() {{ + put("id", "dev001"); + put("type", "sensor"); + }}, + new HashMap() {{ + put("id", "dev002"); + put("type", "actuator"); + }} + )); + inputMap.put("settings", new HashMap() {{ + put("notifications", true); + put("timezone", "UTC-5"); + put("params", new HashMap() {{ + put("param1", "value1"); + put("param2", "value2"); + put("param3", new HashMap() {{ + put("subParam1", "value1"); + put("subParam2", "value2"); + }}); + }}); + }}); + + List excludeList = List.of("age", "id", "param1", "subParam2"); + + HashMap expectedMapWithPath = new HashMap<>(); + expectedMapWithPath.put("name", "Alice"); + expectedMapWithPath.put("devices.0.type", "sensor"); + expectedMapWithPath.put("devices.1.type", "actuator"); + expectedMapWithPath.put("settings.notifications", true); + expectedMapWithPath.put("settings.timezone", "UTC-5"); + expectedMapWithPath.put("settings.params.param2", "value2"); + expectedMapWithPath.put("settings.params.param3.subParam1", "value1"); + + HashMap actualMapWithPaths = new HashMap<>(); + TbUtils.toFlatMap(inputMap, actualMapWithPaths, excludeList, true); + + Assert.assertEquals(expectedMapWithPath, actualMapWithPaths); + + HashMap expectedMapWithoutPaths = new HashMap<>(); + expectedMapWithoutPaths.put("timezone", "UTC-5"); + expectedMapWithoutPaths.put("name", "Alice"); + expectedMapWithoutPaths.put("id", "dev002"); + expectedMapWithoutPaths.put("subParam2", "value2"); + expectedMapWithoutPaths.put("type", "actuator"); + expectedMapWithoutPaths.put("subParam1", "value1"); + expectedMapWithoutPaths.put("param1", "value1"); + expectedMapWithoutPaths.put("notifications", true); + expectedMapWithoutPaths.put("age", 30); + expectedMapWithoutPaths.put("param2", "value2"); + + HashMap actualMapWithoutPaths = new HashMap<>(); + TbUtils.toFlatMap(inputMap, actualMapWithoutPaths, new ArrayList<>(), false); + + Assert.assertEquals(expectedMapWithoutPaths, actualMapWithoutPaths); + } + + + + private static String keyToValue(String key, String extraSymbol) { + return key + "Value" + (extraSymbol == null ? "" : extraSymbol); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From 8f55fdd94b192d7849e381a38636f394be80ec8c Mon Sep 17 00:00:00 2001 From: imbeacon Date: Thu, 20 Apr 2023 17:41:27 +0300 Subject: [PATCH 2/5] Refactoring --- .../thingsboard/script/api/tbel/TbUtils.java | 37 ++++----- .../script/api/tbel/TbUtilsTest.java | 76 +++++++++++++------ 2 files changed, 72 insertions(+), 41 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index c4f86f80a3..f13ff8cad6 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2023 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -97,13 +96,13 @@ public class TbUtils { parserConfig.addImport("bytesToHex", new MethodStub(TbUtils.class.getMethod("bytesToHex", ExecutionArrayList.class))); parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", - Object.class, HashMap.class))); + ExecutionContext.class, Map.class))); parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", - Object.class, HashMap.class, boolean.class))); + ExecutionContext.class, Map.class, boolean.class))); parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", - Object.class, HashMap.class, List.class))); + ExecutionContext.class, Map.class, List.class))); parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", - Object.class, HashMap.class, List.class, boolean.class))); + ExecutionContext.class, Map.class, List.class, boolean.class))); } public static String btoa(String input) { @@ -240,7 +239,7 @@ public class TbUtils { } ExecutionArrayList data = new ExecutionArrayList<>(ctx); for (int i = 0; i < len; i += 2) { - data.add((byte)((Character.digit(hex.charAt(i), 16) << 4) + data.add((byte) ((Character.digit(hex.charAt(i), 16) << 4) + Character.digit(hex.charAt(i + 1), 16))); } return data; @@ -332,23 +331,25 @@ public class TbUtils { return value; } - public static void toFlatMap(Object json, HashMap map) { - toFlatMap(json, map, new ArrayList<>(), true); + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json) { + return toFlatMap(ctx, json, new ArrayList<>(), true); } - public static void toFlatMap(Object json, HashMap map, boolean pathInKey) { - toFlatMap(json, map, new ArrayList<>(), pathInKey); + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json, boolean pathInKey) { + return toFlatMap(ctx, json, new ArrayList<>(), pathInKey); } - public static void toFlatMap(Object json, HashMap map, List excludeList) { - toFlatMap(json, map, excludeList, true); + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json, List excludeList) { + return toFlatMap(ctx, json, excludeList, true); } - public static void toFlatMap(Object json, HashMap map, List excludeList, boolean pathInKey) { + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json, List excludeList, boolean pathInKey) { + ExecutionHashMap map = new ExecutionHashMap<>(16, ctx); parseRecursive(json, map, excludeList, "", pathInKey); + return map; } - private static void parseRecursive(Object json, HashMap map, List excludeList, String path, boolean pathInKey) { + private static void parseRecursive(Object json, Map map, List excludeList, String path, boolean pathInKey) { if (json instanceof Map.Entry) { Map.Entry entry = (Map.Entry) json; if (StringUtils.isNotBlank(path)) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index af10b7b49b..cafef85384 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2023 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,17 +15,50 @@ */ package org.thingsboard.script.api.tbel; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mvel2.ExecutionContext; +import org.mvel2.ParserContext; +import org.mvel2.SandboxedParserConfiguration; +import org.mvel2.execution.ExecutionArrayList; +import org.mvel2.execution.ExecutionHashMap; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Calendar; import java.util.List; -import java.util.Map; +import java.util.Random; + +@Slf4j public class TbUtilsTest { + private ExecutionContext ctx; + + @Before + public void before() { + SandboxedParserConfiguration parserConfig = ParserContext.enableSandboxedMode(); + parserConfig.addImport("JSON", TbJson.class); + parserConfig.registerDataType("Date", TbDate.class, date -> 8L); + parserConfig.registerDataType("Random", Random.class, date -> 8L); + parserConfig.registerDataType("Calendar", Calendar.class, date -> 8L); + try { + TbUtils.register(parserConfig); + } catch (Exception e) { + log.error("Cannot register functions", e); + } + ctx = new ExecutionContext(parserConfig); + Assert.assertNotNull(ctx); + } + + @After + public void after() { + ctx.stop(); + } + @Test public void parseHexToInt() { Assert.assertEquals(0xAB, TbUtils.parseHexToInt("AB")); @@ -91,35 +124,35 @@ public class TbUtilsTest { @Test public void toFlatMap() { - HashMap inputMap = new HashMap<>(); + ExecutionHashMap inputMap = new ExecutionHashMap<>(16, ctx); inputMap.put("name", "Alice"); inputMap.put("age", 30); - inputMap.put("devices", List.of( - new HashMap() {{ + inputMap.put("devices", new ExecutionArrayList<>(List.of( + new ExecutionHashMap<>(16, ctx) {{ put("id", "dev001"); put("type", "sensor"); }}, - new HashMap() {{ + new ExecutionHashMap<>(16, ctx) {{ put("id", "dev002"); put("type", "actuator"); }} - )); - inputMap.put("settings", new HashMap() {{ + ), ctx)); + inputMap.put("settings", new ExecutionHashMap<>(16, ctx) {{ put("notifications", true); put("timezone", "UTC-5"); - put("params", new HashMap() {{ + put("params", new ExecutionHashMap<>(16, ctx) {{ put("param1", "value1"); put("param2", "value2"); - put("param3", new HashMap() {{ + put("param3", new ExecutionHashMap<>(16, ctx) {{ put("subParam1", "value1"); put("subParam2", "value2"); }}); }}); }}); + ExecutionArrayList excludeList = new ExecutionArrayList<>(ctx); + excludeList.addAll(List.of("age", "id", "param1", "subParam2")); - List excludeList = List.of("age", "id", "param1", "subParam2"); - - HashMap expectedMapWithPath = new HashMap<>(); + ExecutionHashMap expectedMapWithPath = new ExecutionHashMap<>(16, ctx); expectedMapWithPath.put("name", "Alice"); expectedMapWithPath.put("devices.0.type", "sensor"); expectedMapWithPath.put("devices.1.type", "actuator"); @@ -128,12 +161,11 @@ public class TbUtilsTest { expectedMapWithPath.put("settings.params.param2", "value2"); expectedMapWithPath.put("settings.params.param3.subParam1", "value1"); - HashMap actualMapWithPaths = new HashMap<>(); - TbUtils.toFlatMap(inputMap, actualMapWithPaths, excludeList, true); + ExecutionHashMap actualMapWithPaths = TbUtils.toFlatMap(ctx, inputMap, excludeList, true); Assert.assertEquals(expectedMapWithPath, actualMapWithPaths); - HashMap expectedMapWithoutPaths = new HashMap<>(); + ExecutionHashMap expectedMapWithoutPaths = new ExecutionHashMap<>(16, ctx); expectedMapWithoutPaths.put("timezone", "UTC-5"); expectedMapWithoutPaths.put("name", "Alice"); expectedMapWithoutPaths.put("id", "dev002"); @@ -145,14 +177,12 @@ public class TbUtilsTest { expectedMapWithoutPaths.put("age", 30); expectedMapWithoutPaths.put("param2", "value2"); - HashMap actualMapWithoutPaths = new HashMap<>(); - TbUtils.toFlatMap(inputMap, actualMapWithoutPaths, new ArrayList<>(), false); + ExecutionHashMap actualMapWithoutPaths = TbUtils.toFlatMap(ctx, inputMap, false); Assert.assertEquals(expectedMapWithoutPaths, actualMapWithoutPaths); } - private static String keyToValue(String key, String extraSymbol) { return key + "Value" + (extraSymbol == null ? "" : extraSymbol); } From 9e98b1c006ebcd2e24d2f5a88e48051b80d6343a Mon Sep 17 00:00:00 2001 From: imbeacon Date: Thu, 20 Apr 2023 17:45:44 +0300 Subject: [PATCH 3/5] License --- .../main/java/org/thingsboard/script/api/tbel/TbUtils.java | 6 +++--- .../java/org/thingsboard/script/api/tbel/TbUtilsTest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index f13ff8cad6..0f9a5cbd2b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2023 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index cafef85384..e445da1c38 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2023 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 7ea35aa95a6d5ba54e47d0b9718013d21ab587f4 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Thu, 20 Apr 2023 18:01:04 +0300 Subject: [PATCH 4/5] License --- .../src/main/java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- .../test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 0f9a5cbd2b..7e7b72d493 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index e445da1c38..82cd74ca30 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 4d46bc11318a40eba8f52320de3d4e03b7b29aa1 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 25 Apr 2023 09:14:48 +0300 Subject: [PATCH 5/5] Added writing the key for arrays --- .../java/org/thingsboard/script/api/tbel/TbUtils.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 7e7b72d493..6c50561363 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -376,7 +376,14 @@ public class TbUtils { if (pathInKey) { map.put(path, json); } else { - String key = path.contains(".") ? path.substring(path.lastIndexOf('.') + 1) : path; + String key = path.substring(path.lastIndexOf('.') + 1); + if (StringUtils.isNumeric(key)) { + int pos = path.length(); + for (int i = 0; i < 2; i++) { + pos = path.lastIndexOf('.', pos - 1); + } + key = path.substring(pos + 1); + } map.put(key, json); } }