Merge pull request #7240 from YuriyLytvynchuk/feature/node_copy_from_metadata_to_msg

[3.4.2] Feature: New RuleNode 'copy keys'

rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbCopyKeysNode.java

@@ -0,0 +1,107 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.rule.engine.transform;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.extern.slf4j.Slf4j;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.rule.engine.api.RuleNode;
+import org.thingsboard.rule.engine.api.TbContext;
+import org.thingsboard.rule.engine.api.TbNode;
+import org.thingsboard.rule.engine.api.TbNodeConfiguration;
+import org.thingsboard.rule.engine.api.TbNodeException;
+import org.thingsboard.rule.engine.api.util.TbNodeUtils;
+import org.thingsboard.server.common.data.plugin.ComponentType;
+import org.thingsboard.server.common.msg.TbMsg;
+import org.thingsboard.server.common.msg.TbMsgMetaData;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
+
+@Slf4j
+@RuleNode(
+        type = ComponentType.TRANSFORMATION,
+        name = "copy keys",
+        configClazz = TbCopyKeysNodeConfiguration.class,
+        nodeDescription = "Copies the msg or metadata keys with specified key names selected in the list",
+        nodeDetails = "Will fetch fields values specified in list. If specified field is not part of msg or metadata fields it will be ignored." +
+                "Returns transformed messages via <code>Success</code> chain",
+        uiResources = {"static/rulenode/rulenode-core-config.js"},
+        configDirective = "tbTransformationNodeCopyKeysConfig",
+        icon = "content_copy"
+)
+public class TbCopyKeysNode implements TbNode {
+
+    private TbCopyKeysNodeConfiguration config;
+    private List<Pattern> patternKeys;
+    private boolean fromMetadata;
+
+    @Override
+    public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
+        this.config = TbNodeUtils.convert(configuration, TbCopyKeysNodeConfiguration.class);
+        this.fromMetadata = config.isFromMetadata();
+        this.patternKeys = new ArrayList<>();
+        config.getKeys().forEach(key -> {
+            this.patternKeys.add(Pattern.compile(key));
+        });
+    }
+
+    @Override
+    public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException {
+        TbMsgMetaData metaData = msg.getMetaData();
+        String msgData = msg.getData();
+        boolean msgChanged = false;
+        JsonNode dataNode = JacksonUtil.toJsonNode(msgData);
+        if (dataNode.isObject()) {
+            if (fromMetadata) {
+                ObjectNode msgDataNode = (ObjectNode) dataNode;
+                Map<String, String> metaDataMap = metaData.getData();
+                for (Map.Entry<String, String> entry : metaDataMap.entrySet()) {
+                    String keyData = entry.getKey();
+                    if (checkKey(keyData)) {
+                        msgChanged = true;
+                        msgDataNode.put(keyData, entry.getValue());
+                    }
+                }
+                msgData = JacksonUtil.toString(msgDataNode);
+            } else {
+                Iterator<Map.Entry<String, JsonNode>> iteratorNode = dataNode.fields();
+                while (iteratorNode.hasNext()) {
+                    Map.Entry<String, JsonNode> entry = iteratorNode.next();
+                    String keyData = entry.getKey();
+                    if (checkKey(keyData)) {
+                        msgChanged = true;
+                        metaData.putValue(keyData, JacksonUtil.toString(entry.getValue()));
+                    }
+                }
+            }
+        }
+        if (msgChanged) {
+            ctx.tellSuccess(TbMsg.transformMsg(msg, msg.getType(), msg.getOriginator(), metaData, msgData));
+        } else {
+            ctx.tellSuccess(msg);
+        }
+    }
+
+    boolean checkKey(String key) {
+        return patternKeys.stream().anyMatch(pattern -> pattern.matcher(key).matches());
+    }
+}

rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbCopyKeysNodeConfiguration.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.rule.engine.transform;
+
+import lombok.Data;
+import org.thingsboard.rule.engine.api.NodeConfiguration;
+
+import java.util.Collections;
+import java.util.Set;
+
+@Data
+public class TbCopyKeysNodeConfiguration implements NodeConfiguration<TbCopyKeysNodeConfiguration> {
+
+    private boolean fromMetadata;
+    private Set<String> keys;
+
+    @Override
+    public TbCopyKeysNodeConfiguration defaultConfiguration() {
+        TbCopyKeysNodeConfiguration configuration = new TbCopyKeysNodeConfiguration();
+        configuration.setKeys(Collections.emptySet());
+        configuration.setFromMetadata(false);
+        return configuration;
+    }
+
+}

rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbCopyKeysNodeTest.java

@@ -0,0 +1,166 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.rule.engine.transform;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.rule.engine.api.TbContext;
+import org.thingsboard.rule.engine.api.TbNodeConfiguration;
+import org.thingsboard.rule.engine.api.TbNodeException;
+import org.thingsboard.server.common.data.id.DeviceId;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.msg.TbMsg;
+import org.thingsboard.server.common.msg.TbMsgMetaData;
+import org.thingsboard.server.common.msg.queue.TbMsgCallback;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class TbCopyKeysNodeTest {
+    final ObjectMapper mapper = new ObjectMapper();
+
+    DeviceId deviceId;
+    TbCopyKeysNode node;
+    TbCopyKeysNodeConfiguration config;
+    TbNodeConfiguration nodeConfiguration;
+    TbContext ctx;
+    TbMsgCallback callback;
+
+    @BeforeEach
+    void setUp() throws TbNodeException {
+        deviceId = new DeviceId(UUID.randomUUID());
+        callback = mock(TbMsgCallback.class);
+        ctx = mock(TbContext.class);
+        config = new TbCopyKeysNodeConfiguration().defaultConfiguration();
+        config.setKeys(Set.of("TestKey_1", "TestKey_2", "TestKey_3", "(\\w*)Data(\\w*)"));
+        config.setFromMetadata(true);
+        nodeConfiguration = new TbNodeConfiguration(mapper.valueToTree(config));
+        node = spy(new TbCopyKeysNode());
+        node.init(ctx, nodeConfiguration);
+    }
+
+    @AfterEach
+    void tearDown() {
+        node.destroy();
+    }
+
+    @Test
+    void givenDefaultConfig_whenVerify_thenOK() {
+        TbCopyKeysNodeConfiguration defaultConfig = new TbCopyKeysNodeConfiguration().defaultConfiguration();
+        assertThat(defaultConfig.getKeys()).isEqualTo(Collections.emptySet());
+        assertThat(defaultConfig.isFromMetadata()).isEqualTo(false);
+    }
+
+    @Test
+    void givenMsgFromMetadata_whenOnMsg_thenVerifyOutput() throws Exception {
+        String data = "{}";
+        node.onMsg(ctx, getTbMsg(deviceId, data));
+
+        ArgumentCaptor<TbMsg> newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class);
+        verify(ctx, times(1)).tellSuccess(newMsgCaptor.capture());
+        verify(ctx, never()).tellFailure(any(), any());
+
+        TbMsg newMsg = newMsgCaptor.getValue();
+        assertThat(newMsg).isNotNull();
+
+        JsonNode dataNode = JacksonUtil.toJsonNode(newMsg.getData());
+        assertThat(dataNode.has("TestKey_1")).isEqualTo(true);
+        assertThat(dataNode.has("voltageDataValue")).isEqualTo(true);
+    }
+
+    @Test
+    void givenMsgFromMsg_whenOnMsg_thenVerifyOutput() throws Exception {
+        config.setFromMetadata(false);
+        nodeConfiguration = new TbNodeConfiguration(mapper.valueToTree(config));
+        node.init(ctx, nodeConfiguration);
+
+        String data = "{\"DigitData\":22.5,\"TempDataValue\":10.5}";
+        node.onMsg(ctx, getTbMsg(deviceId, data));
+
+        ArgumentCaptor<TbMsg> newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class);
+        verify(ctx, times(1)).tellSuccess(newMsgCaptor.capture());
+        verify(ctx, never()).tellFailure(any(), any());
+
+        TbMsg newMsg = newMsgCaptor.getValue();
+        assertThat(newMsg).isNotNull();
+
+        Map<String, String> metaDataMap = newMsg.getMetaData().getData();
+        assertThat(metaDataMap.containsKey("DigitData")).isEqualTo(true);
+        assertThat(metaDataMap.containsKey("TempDataValue")).isEqualTo(true);
+    }
+
+    @Test
+    void givenEmptyKeys_whenOnMsg_thenVerifyOutput() throws Exception {
+        TbCopyKeysNodeConfiguration defaultConfig = new TbCopyKeysNodeConfiguration().defaultConfiguration();
+        nodeConfiguration = new TbNodeConfiguration(mapper.valueToTree(defaultConfig));
+        node.init(ctx, nodeConfiguration);
+
+        String data = "{\"DigitData\":22.5,\"TempDataValue\":10.5}";
+        TbMsg msg = getTbMsg(deviceId, data);
+        node.onMsg(ctx, msg);
+
+        ArgumentCaptor<TbMsg> newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class);
+        verify(ctx, times(1)).tellSuccess(newMsgCaptor.capture());
+        verify(ctx, never()).tellFailure(any(), any());
+
+        TbMsg newMsg = newMsgCaptor.getValue();
+        assertThat(newMsg).isNotNull();
+
+        assertThat(newMsg.getMetaData()).isEqualTo(msg.getMetaData());
+    }
+
+    @Test
+    void givenMsgDataNotJSONObject_whenOnMsg_thenTVerifyOutput() throws Exception {
+        String data = "[]";
+        TbMsg msg = getTbMsg(deviceId, data);
+        node.onMsg(ctx, msg);
+
+        ArgumentCaptor<TbMsg> newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class);
+        verify(ctx, times(1)).tellSuccess(newMsgCaptor.capture());
+        verify(ctx, never()).tellFailure(any(), any());
+
+        TbMsg newMsg = newMsgCaptor.getValue();
+        assertThat(newMsg).isNotNull();
+
+        assertThat(newMsg).isSameAs(msg);
+    }
+
+    private TbMsg getTbMsg(EntityId entityId, String data) {
+        final Map<String, String> mdMap = Map.of(
+                "TestKey_1", "Test",
+                "country", "US",
+                "voltageDataValue", "220",
+                "city", "NY"
+        );
+        return TbMsg.newMsg("POST_ATTRIBUTES_REQUEST", entityId, new TbMsgMetaData(mdMap), data, callback);
+    }
+
+}