Merge with master

application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@@ -65,8 +65,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
     private final TbNotificationEntityService entityNotificationService;
 
     protected static final List<EntityType> SUPPORTED_ENTITY_TYPES = List.of(
-            EntityType.CUSTOMER, EntityType.ASSET_PROFILE, EntityType.ASSET, EntityType.RULE_CHAIN,
-            EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE,
+            EntityType.CUSTOMER, EntityType.RULE_CHAIN, EntityType.DASHBOARD,
+            EntityType.ASSET_PROFILE, EntityType.ASSET,
+            EntityType.DEVICE_PROFILE, EntityType.DEVICE,
             EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE,
             EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE
     );

application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java

@@ -26,6 +26,7 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
 
 import java.util.Collection;
 import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.UUID;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -47,8 +48,8 @@ public abstract class BaseEntityExportService<I extends EntityId, E extends Expo
 
     public abstract Set<EntityType> getSupportedEntityTypes();
 
-    protected void replaceUuidsRecursively(EntitiesExportCtx<?> ctx, JsonNode node, Set<String> skipFieldsSet) {
-        JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid));
+    protected void replaceUuidsRecursively(EntitiesExportCtx<?> ctx, JsonNode node, Set<String> skipFieldsSet, Pattern includedFieldsPattern) {
+        JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, includedFieldsPattern, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid));
     }
 
     protected Stream<UUID> toExternalIds(Collection<UUID> internalIds, Function<UUID, EntityId> entityIdCreator,

application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java

@@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
 
-import java.util.Collections;
 import java.util.Set;
 
 @Service
@@ -41,10 +40,10 @@ public class DashboardExportService extends BaseEntityExportService<DashboardId,
             });
         }
         for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) {
-            replaceUuidsRecursively(ctx, entityAlias, Collections.emptySet());
+            replaceUuidsRecursively(ctx, entityAlias, Set.of("id"), null);
         }
         for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) {
-            replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id"));
+            replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Set.of("id"), null);
         }
     }
 

application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java

@@ -30,6 +30,8 @@ import java.util.Collections;
 import java.util.Optional;
 import java.util.Set;
 
+import static org.thingsboard.server.service.sync.ie.importing.impl.RuleChainImportService.PROCESSED_CONFIG_FIELDS_PATTERN;
+
 @Service
 @TbCoreComponent
 @RequiredArgsConstructor
@@ -47,7 +49,7 @@ public class RuleChainExportService extends BaseEntityExportService<RuleChainId,
                     ruleNode.setId(ctx.getExternalId(ruleNode.getId()));
                     ruleNode.setCreatedTime(0);
                     ruleNode.setExternalId(null);
-                    replaceUuidsRecursively(ctx, ruleNode.getConfiguration(), Collections.emptySet());
+                    replaceUuidsRecursively(ctx, ruleNode.getConfiguration(), Collections.emptySet(), PROCESSED_CONFIG_FIELDS_PATTERN);
                 });
         Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList())
                 .forEach(ruleChainConnectionInfo -> {

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@@ -65,6 +65,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -386,9 +387,12 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
         return oldEntity == null ? null : getter.apply(oldEntity);
     }
 
-    protected void replaceIdsRecursively(EntitiesImportCtx ctx, IdProvider idProvider, JsonNode entityAlias, Set<String> skipFieldsSet, LinkedHashSet<EntityType> hints) {
-        JacksonUtil.replaceUuidsRecursively(entityAlias, skipFieldsSet,
-                uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints).map(EntityId::getId).orElse(uuid));
+    protected void replaceIdsRecursively(EntitiesImportCtx ctx, IdProvider idProvider, JsonNode json,
+                                         Set<String> skipFieldsSet, Pattern includedFieldsPattern,
+                                         LinkedHashSet<EntityType> hints) {
+        JacksonUtil.replaceUuidsRecursively(json, skipFieldsSet, includedFieldsPattern,
+                uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints)
+                        .map(EntityId::getId).orElse(uuid));
     }
 
 }

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java

@@ -65,10 +65,10 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
     @Override
     protected Dashboard prepare(EntitiesImportCtx ctx, Dashboard dashboard, Dashboard old, EntityExportData<Dashboard> exportData, IdProvider idProvider) {
         for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) {
-            replaceIdsRecursively(ctx, idProvider, entityAlias, Set.of("id"), HINTS);
+            replaceIdsRecursively(ctx, idProvider, entityAlias, Set.of("id"), null, HINTS);
         }
         for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) {
-            replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Set.of("id"), HINTS);
+            replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Set.of("id"), null, HINTS);
         }
         return dashboard;
     }

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@@ -42,6 +42,7 @@ import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -51,6 +52,7 @@ import java.util.stream.Collectors;
 public class RuleChainImportService extends BaseEntityImportService<RuleChainId, RuleChain, RuleChainExportData> {
 
     private static final LinkedHashSet<EntityType> HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET));
+    public static final Pattern PROCESSED_CONFIG_FIELDS_PATTERN = Pattern.compile(".*[iI]d.*");
 
     private final TbRuleChainService tbRuleChainService;
     private final RuleChainService ruleChainService;
@@ -91,7 +93,8 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
             });
         }
 
-        ruleNodes.forEach(ruleNode -> replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(), Collections.emptySet(), HINTS));
+        ruleNodes.forEach(ruleNode -> replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(),
+                Collections.emptySet(), PROCESSED_CONFIG_FIELDS_PATTERN, HINTS));
         Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList())
                 .forEach(ruleChainConnectionInfo -> {
                     ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false));

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java

@@ -103,7 +103,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
     private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>();
     private final Map<UUID, HashMap<Integer, String[]>> chunkedMsgs = new ConcurrentHashMap<>();
 
-    @Value("${queue.vc.request-timeout:60000}")
+    @Value("${queue.vc.request-timeout:180000}")
     private int requestTimeout;
     @Value("${queue.vc.msg-chunk-size:500000}")
     private int msgChunkSize;

application/src/main/resources/thingsboard.yml

@@ -493,7 +493,7 @@ cache:
       timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}"
       maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}"
     versionControlTask:
-      timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}"
+      timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:20}"
       maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}"
     userSettings:
       timeToLiveInMinutes: "${CACHE_SPECS_USER_SETTINGS_TTL:1440}"
@@ -1211,8 +1211,8 @@ queue:
     topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"
     partitions: "${TB_QUEUE_VC_PARTITIONS:10}"
     poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}"
-    pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}"
-    request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}"
+    pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}"
+    request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:180000}"
     msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}"
   js:
     # JS Eval request topic

application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java

@@ -53,6 +53,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainMetaData;
 import org.thingsboard.server.common.data.rule.RuleNode;
+import org.thingsboard.server.common.data.script.ScriptLanguage;
 import org.thingsboard.server.common.data.security.DeviceCredentials;
 import org.thingsboard.server.common.data.sync.ie.DeviceExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
@@ -65,9 +66,11 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
 import org.thingsboard.server.service.action.EntityActionService;
 import org.thingsboard.server.service.ota.OtaPackageStateService;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -338,6 +341,47 @@ public class ExportImportServiceSqlTest extends BaseExportImportServiceTest {
         checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData);
     }
 
+    @Test
+    public void testImportRuleChain_ruleNodesConfigs() throws Exception {
+        Customer customer = createCustomer(tenantId1, "Customer 1");
+        RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1");
+        RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId());
+
+        List<RuleNode> nodes = new ArrayList<>(metaData.getNodes());
+        RuleNode generatorNode = new RuleNode();
+        generatorNode.setName("Generator");
+        generatorNode.setType(TbMsgGeneratorNode.class.getName());
+        TbMsgGeneratorNodeConfiguration generatorNodeConfig = new TbMsgGeneratorNodeConfiguration();
+        generatorNodeConfig.setOriginatorType(EntityType.ASSET_PROFILE);
+        generatorNodeConfig.setOriginatorId(customer.getId().toString());
+        generatorNodeConfig.setPeriodInSeconds(5);
+        generatorNodeConfig.setMsgCount(1);
+        generatorNodeConfig.setScriptLang(ScriptLanguage.JS);
+        UUID someUuid = UUID.randomUUID();
+        generatorNodeConfig.setJsScript("var msg = { temp: 42, humidity: 77 };\n" +
+                "var metadata = { data: 40 };\n" +
+                "var msgType = \"POST_TELEMETRY_REQUEST\";\n" +
+                "var someUuid = \"" + someUuid + "\";\n" +
+                "return { msg: msg, metadata: metadata, msgType: msgType };");
+        generatorNode.setConfiguration(mapper.valueToTree(generatorNodeConfig));
+        nodes.add(generatorNode);
+        metaData.setNodes(nodes);
+        ruleChainService.saveRuleChainMetaData(tenantId1, metaData);
+
+        EntityExportData<RuleChain> ruleChainExportData = exportEntity(tenantAdmin1, ruleChain.getId());
+        EntityExportData<Customer> customerExportData = exportEntity(tenantAdmin1, customer.getId());
+
+        Customer importedCustomer = importEntity(tenantAdmin2, customerExportData).getSavedEntity();
+        RuleChain importedRuleChain = importEntity(tenantAdmin2, ruleChainExportData).getSavedEntity();
+        RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId());
+
+        TbMsgGeneratorNodeConfiguration importedGeneratorNodeConfig = JacksonUtil.treeToValue(importedMetaData.getNodes().stream()
+                .filter(node -> node.getName().equals(generatorNode.getName()))
+                .findFirst().get().getConfiguration(), TbMsgGeneratorNodeConfiguration.class);
+        assertThat(importedGeneratorNodeConfig.getOriginatorId()).isEqualTo(importedCustomer.getId().toString());
+        assertThat(importedGeneratorNodeConfig.getJsScript()).contains("var someUuid = \"" + someUuid + "\";");
+    }
+
 
     @Test
     public void testExportImportWithInboundRelations_betweenTenants() throws Exception {

common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java

@@ -44,6 +44,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import java.util.function.UnaryOperator;
+import java.util.regex.Pattern;
 
 /**
  * Created by Valerii Sosliuk on 5/12/2017.
@@ -227,7 +228,7 @@ public class JacksonUtil {
         return node;
     }
 
-    public static void replaceUuidsRecursively(JsonNode node, Set<String> skipFieldsSet, UnaryOperator<UUID> replacer) {
+    public static void replaceUuidsRecursively(JsonNode node, Set<String> skipFieldsSet, Pattern includedFieldsPattern, UnaryOperator<UUID> replacer) {
         if (node == null) {
             return;
         }
@@ -239,9 +240,14 @@ public class JacksonUtil {
                 if (skipFieldsSet.contains(fieldName)) {
                     continue;
                 }
+                if (includedFieldsPattern != null) {
+                    if (!RegexUtils.matches(fieldName, includedFieldsPattern)) {
+                        continue;
+                    }
+                }
                 var child = objectNode.get(fieldName);
                 if (child.isObject() || child.isArray()) {
-                    replaceUuidsRecursively(child, skipFieldsSet, replacer);
+                    replaceUuidsRecursively(child, skipFieldsSet, includedFieldsPattern, replacer);
                 } else if (child.isTextual()) {
                     String text = child.asText();
                     String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString());
@@ -255,7 +261,7 @@ public class JacksonUtil {
             for (int i = 0; i < array.size(); i++) {
                 JsonNode arrayElement = array.get(i);
                 if (arrayElement.isObject() || arrayElement.isArray()) {
-                    replaceUuidsRecursively(arrayElement, skipFieldsSet, replacer);
+                    replaceUuidsRecursively(arrayElement, skipFieldsSet, includedFieldsPattern, replacer);
                 } else if (arrayElement.isTextual()) {
                     String text = arrayElement.asText();
                     String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString());

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java

@@ -122,7 +122,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
 
     @Value("${queue.vc.poll-interval:25}")
     private long pollDuration;
-    @Value("${queue.vc.pack-processing-timeout:60000}")
+    @Value("${queue.vc.pack-processing-timeout:180000}")
     private long packProcessingTimeout;
     @Value("${vc.git.io_pool_size:3}")
     private int ioPoolSize;

msa/vc-executor/src/main/resources/tb-vc-executor.yml

@@ -168,7 +168,7 @@ queue:
     topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"
     partitions: "${TB_QUEUE_VC_PARTITIONS:10}"
     poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}"
-    pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}"
+    pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}"
     msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}"
 
 vc: