From 5f7f57cb636cba85acf5a73a89057ef43684dc61 Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 16 Apr 2024 17:33:12 +0300 Subject: [PATCH 1/6] tbel: encodeURI() decodeURI() == mdn --- .../thingsboard/script/api/tbel/TbUtils.java | 63 ++++++++++++++++--- .../script/api/tbel/TbUtilsTest.java | 13 +++- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index bbcdd1bf00..1cf88ce3f9 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -16,6 +16,7 @@ package org.thingsboard.script.api.tbel; import com.google.common.primitives.Bytes; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; import org.mvel2.ExecutionContext; import org.mvel2.ParserConfiguration; @@ -29,32 +30,50 @@ import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; +import java.net.URLDecoder; +import java.net.URLEncoder; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; +@Slf4j public class TbUtils { private static final byte[] HEX_ARRAY = "0123456789ABCDEF".getBytes(StandardCharsets.US_ASCII); + private static final LinkedHashMap enCodeMdn = new LinkedHashMap<>(); + + static { + enCodeMdn.put("\\+", "%20"); + enCodeMdn.put("%21", "!"); + enCodeMdn.put("%27", "'"); + enCodeMdn.put("%28", "\\("); + enCodeMdn.put("%29", "\\)"); + enCodeMdn.put("%7E", "~"); + enCodeMdn.put("%3B", ";"); + enCodeMdn.put("%2C", ","); + enCodeMdn.put("%2F", "/"); + enCodeMdn.put("%3F", "\\?"); + enCodeMdn.put("%3A", ":"); + enCodeMdn.put("%40", "@"); + enCodeMdn.put("%26", "&"); + enCodeMdn.put("%3D", "="); + enCodeMdn.put("%2B", "\\+"); + enCodeMdn.put("%24", Matcher.quoteReplacement("$")); + enCodeMdn.put("%23", "#"); + } + public static void register(ParserConfiguration parserConfig) throws Exception { parserConfig.addImport("btoa", new MethodStub(TbUtils.class.getMethod("btoa", String.class))); @@ -175,6 +194,10 @@ public class TbUtils { ExecutionContext.class, Map.class, List.class))); parserConfig.addImport("toFlatMap", new MethodStub(TbUtils.class.getMethod("toFlatMap", ExecutionContext.class, Map.class, List.class, boolean.class))); + parserConfig.addImport("encodeURI", new MethodStub(TbUtils.class.getMethod("encodeURI", + String.class))); + parserConfig.addImport("decodeURI", new MethodStub(TbUtils.class.getMethod("decodeURI", + String.class))); } public static String btoa(String input) { @@ -623,6 +646,26 @@ public class TbUtils { return map; } + public static String encodeURI(String uri) { + String encoded = URLEncoder.encode(uri, StandardCharsets.UTF_8); + Optional encodedMdnOpt = Optional.of(encoded); + for (var entry : enCodeMdn.entrySet()) { + encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(entry.getKey(), entry.getValue())); + } + return encodedMdnOpt.orElse(null); + } + + public static String decodeURI(String uri) { + ArrayList alKeys = new ArrayList<>(enCodeMdn.keySet()); + Collections.reverse(alKeys); + Optional encodedMdnOpt = Optional.of(uri); + for (String strKey : alKeys) { + encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(enCodeMdn.get(strKey), strKey)); + enCodeMdn.get(strKey); + } + return encodedMdnOpt.orElse(null) == null ? null : URLDecoder.decode(encodedMdnOpt.orElse(null), StandardCharsets.UTF_8); + } + private static void parseRecursive(Object json, Map map, List excludeList, String path, boolean pathInKey) { if (json instanceof Map.Entry) { Map.Entry entry = (Map.Entry) json; diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index c140559a0f..8325e69f66 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -31,13 +31,11 @@ import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Calendar; import java.util.List; import java.util.Random; @Slf4j - public class TbUtilsTest { private ExecutionContext ctx; @@ -474,7 +472,16 @@ public class TbUtilsTest { } } - + @Test + public void encodeDecodeUri_Test() { + String uriOriginal = "-_.!~*'();/?:@&=+$,#ht://example.ж д a/path with spaces/?param1=Київ 1¶m2=Україна2"; + String uriEncodeExpected = "-_.!~*'();/?:@&=+$,#ht://example.%D0%B6%20%D0%B4%20a/path%20with%20spaces/?param1=%D0%9A%D0%B8%D1%97%D0%B2%201¶m2=%D0%A3%D0%BA%D1%80%D0%B0%D1%97%D0%BD%D0%B02"; + String uriEncodeActual = TbUtils.encodeURI(uriOriginal); + Assert.assertEquals(uriEncodeExpected, uriEncodeActual); + + String uriDecodeActual = TbUtils.decodeURI(uriEncodeActual); + Assert.assertEquals(uriOriginal, uriDecodeActual); + } private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From e488f71444698c6cd8d2f35317fc02de64f65f10 Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 16 Apr 2024 17:56:18 +0300 Subject: [PATCH 2/6] tbel: encodeURI() decodeURI() == mdn 1 --- .../src/main/java/org/thingsboard/script/api/tbel/TbUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 1cf88ce3f9..63019c6da1 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -661,7 +661,6 @@ public class TbUtils { Optional encodedMdnOpt = Optional.of(uri); for (String strKey : alKeys) { encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(enCodeMdn.get(strKey), strKey)); - enCodeMdn.get(strKey); } return encodedMdnOpt.orElse(null) == null ? null : URLDecoder.decode(encodedMdnOpt.orElse(null), StandardCharsets.UTF_8); } From 2923c9dbd39121f1b79bcb85cbf72dd20f56f4c5 Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 16 Apr 2024 18:17:30 +0300 Subject: [PATCH 3/6] tbel: encodeURI() decodeURI() == mdn 2 --- .../src/main/java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 63019c6da1..8d3bae3e09 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -662,7 +662,7 @@ public class TbUtils { for (String strKey : alKeys) { encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(enCodeMdn.get(strKey), strKey)); } - return encodedMdnOpt.orElse(null) == null ? null : URLDecoder.decode(encodedMdnOpt.orElse(null), StandardCharsets.UTF_8); + return encodedMdnOpt.orElse(null) == null ? null : URLDecoder.decode(encodedMdnOpt.get(), StandardCharsets.UTF_8); } private static void parseRecursive(Object json, Map map, List excludeList, String path, boolean pathInKey) { From 10aa7ae795eaa9644b633c9d240627ac2a2b74c1 Mon Sep 17 00:00:00 2001 From: nick Date: Wed, 17 Apr 2024 11:12:43 +0300 Subject: [PATCH 4/6] tbel: encodeURI() decodeURI() == mdn 3 --- .../src/main/java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 8d3bae3e09..1ec62aacd6 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -662,7 +662,7 @@ public class TbUtils { for (String strKey : alKeys) { encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(enCodeMdn.get(strKey), strKey)); } - return encodedMdnOpt.orElse(null) == null ? null : URLDecoder.decode(encodedMdnOpt.get(), StandardCharsets.UTF_8); + return encodedMdnOpt.map(s -> URLDecoder.decode(s, StandardCharsets.UTF_8)).orElse(null); } private static void parseRecursive(Object json, Map map, List excludeList, String path, boolean pathInKey) { From 332c67631767414fccdff68b4a57e69c7a102b22 Mon Sep 17 00:00:00 2001 From: nick Date: Wed, 17 Apr 2024 11:50:51 +0300 Subject: [PATCH 5/6] tbel: encodeURI() decodeURI() comments 1 --- .../thingsboard/script/api/tbel/TbUtils.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 1ec62aacd6..209df886b5 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -52,26 +52,26 @@ public class TbUtils { private static final byte[] HEX_ARRAY = "0123456789ABCDEF".getBytes(StandardCharsets.US_ASCII); - private static final LinkedHashMap enCodeMdn = new LinkedHashMap<>(); + private static final LinkedHashMap mdnEncodingReplacements = new LinkedHashMap<>(); static { - enCodeMdn.put("\\+", "%20"); - enCodeMdn.put("%21", "!"); - enCodeMdn.put("%27", "'"); - enCodeMdn.put("%28", "\\("); - enCodeMdn.put("%29", "\\)"); - enCodeMdn.put("%7E", "~"); - enCodeMdn.put("%3B", ";"); - enCodeMdn.put("%2C", ","); - enCodeMdn.put("%2F", "/"); - enCodeMdn.put("%3F", "\\?"); - enCodeMdn.put("%3A", ":"); - enCodeMdn.put("%40", "@"); - enCodeMdn.put("%26", "&"); - enCodeMdn.put("%3D", "="); - enCodeMdn.put("%2B", "\\+"); - enCodeMdn.put("%24", Matcher.quoteReplacement("$")); - enCodeMdn.put("%23", "#"); + mdnEncodingReplacements.put("\\+", "%20"); + mdnEncodingReplacements.put("%21", "!"); + mdnEncodingReplacements.put("%27", "'"); + mdnEncodingReplacements.put("%28", "\\("); + mdnEncodingReplacements.put("%29", "\\)"); + mdnEncodingReplacements.put("%7E", "~"); + mdnEncodingReplacements.put("%3B", ";"); + mdnEncodingReplacements.put("%2C", ","); + mdnEncodingReplacements.put("%2F", "/"); + mdnEncodingReplacements.put("%3F", "\\?"); + mdnEncodingReplacements.put("%3A", ":"); + mdnEncodingReplacements.put("%40", "@"); + mdnEncodingReplacements.put("%26", "&"); + mdnEncodingReplacements.put("%3D", "="); + mdnEncodingReplacements.put("%2B", "\\+"); + mdnEncodingReplacements.put("%24", Matcher.quoteReplacement("$")); + mdnEncodingReplacements.put("%23", "#"); } public static void register(ParserConfiguration parserConfig) throws Exception { @@ -649,18 +649,18 @@ public class TbUtils { public static String encodeURI(String uri) { String encoded = URLEncoder.encode(uri, StandardCharsets.UTF_8); Optional encodedMdnOpt = Optional.of(encoded); - for (var entry : enCodeMdn.entrySet()) { + for (var entry : mdnEncodingReplacements.entrySet()) { encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(entry.getKey(), entry.getValue())); } return encodedMdnOpt.orElse(null); } public static String decodeURI(String uri) { - ArrayList alKeys = new ArrayList<>(enCodeMdn.keySet()); - Collections.reverse(alKeys); + ArrayList allKeys = new ArrayList<>(mdnEncodingReplacements.keySet()); + Collections.reverse(allKeys); Optional encodedMdnOpt = Optional.of(uri); - for (String strKey : alKeys) { - encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(enCodeMdn.get(strKey), strKey)); + for (String strKey : allKeys) { + encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(mdnEncodingReplacements.get(strKey), strKey)); } return encodedMdnOpt.map(s -> URLDecoder.decode(s, StandardCharsets.UTF_8)).orElse(null); } From 4e9d03fce4d5b40480b6a4d56dff25f15a2eacb5 Mon Sep 17 00:00:00 2001 From: nick Date: Wed, 17 Apr 2024 13:09:54 +0300 Subject: [PATCH 6/6] tbel: encodeURI() decodeURI() comments 2 --- .../thingsboard/script/api/tbel/TbUtils.java | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 209df886b5..283044b279 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -43,7 +43,6 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; @@ -612,22 +611,6 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(precision, RoundingMode.HALF_UP).floatValue(); } - private static boolean isHexadecimal(String value) { - return value != null && (value.contains("0x") || value.contains("0X")); - } - - private static String prepareNumberString(String value) { - if (value != null) { - value = value.trim(); - if (isHexadecimal(value)) { - value = value.replace("0x", ""); - value = value.replace("0X", ""); - } - value = value.replace(",", "."); - } - return value; - } - public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json) { return toFlatMap(ctx, json, new ArrayList<>(), true); } @@ -646,23 +629,22 @@ public class TbUtils { return map; } + public static String encodeURI(String uri) { String encoded = URLEncoder.encode(uri, StandardCharsets.UTF_8); - Optional encodedMdnOpt = Optional.of(encoded); for (var entry : mdnEncodingReplacements.entrySet()) { - encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(entry.getKey(), entry.getValue())); + encoded = encoded.replaceAll(entry.getKey(), entry.getValue()); } - return encodedMdnOpt.orElse(null); + return encoded; } public static String decodeURI(String uri) { ArrayList allKeys = new ArrayList<>(mdnEncodingReplacements.keySet()); Collections.reverse(allKeys); - Optional encodedMdnOpt = Optional.of(uri); for (String strKey : allKeys) { - encodedMdnOpt = Optional.of(encodedMdnOpt.get().replaceAll(mdnEncodingReplacements.get(strKey), strKey)); + uri = uri.replaceAll(mdnEncodingReplacements.get(strKey), strKey); } - return encodedMdnOpt.map(s -> URLDecoder.decode(s, StandardCharsets.UTF_8)).orElse(null); + return URLDecoder.decode(uri, StandardCharsets.UTF_8); } private static void parseRecursive(Object json, Map map, List excludeList, String path, boolean pathInKey) { @@ -705,6 +687,22 @@ public class TbUtils { } } + private static boolean isHexadecimal(String value) { + return value != null && (value.contains("0x") || value.contains("0X")); + } + + private static String prepareNumberString(String value) { + if (value != null) { + value = value.trim(); + if (isHexadecimal(value)) { + value = value.replace("0x", ""); + value = value.replace("0X", ""); + } + value = value.replace(",", "."); + } + return value; + } + private static boolean isValidRadix(String value, int radix) { for (int i = 0; i < value.length(); i++) { if (i == 0 && value.charAt(i) == '-') {