remove null chars from str value of attributes and telemetry for sql

application/src/main/resources/thingsboard.yml

@@ -204,6 +204,8 @@ sql:
       batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}"
       batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
       stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:1000}"
+    # Specify whether to remove null characters from strValue of attributes and timeseries before insert
+    remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
 
 # Actor system parameters
 actors:

dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java

@@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.attributes;
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.jdbc.core.BatchPreparedStatementSetter;
 import org.springframework.jdbc.core.JdbcTemplate;
@@ -34,12 +35,16 @@ import java.sql.SQLException;
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Pattern;
 
 @SqlDao
 @Repository
 @Slf4j
 public abstract class AttributeKvInsertRepository {
 
+    private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE)));
+    private static final String EMPTY_STR = "";
+
     private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, last_update_ts = ? " +
             "WHERE entity_type = ? and entity_id = ? and attribute_type =? and attribute_key = ?;";
 
@@ -60,6 +65,9 @@ public abstract class AttributeKvInsertRepository {
     @Autowired
     private TransactionTemplate transactionTemplate;
 
+    @Value("${sql.remove_null_chars}")
+    private boolean removeNullChars;
+
     @PersistenceContext
     protected EntityManager entityManager;
 
@@ -99,7 +107,7 @@ public abstract class AttributeKvInsertRepository {
                 .setParameter("entity_id", entity.getId().getEntityId())
                 .setParameter("attribute_type", entity.getId().getAttributeType())
                 .setParameter("attribute_key", entity.getId().getAttributeKey())
-                .setParameter("str_v", entity.getStrValue())
+                .setParameter("str_v", replaceNullChars(entity.getStrValue()))
                 .setParameter("last_update_ts", entity.getLastUpdateTs())
                 .executeUpdate();
     }
@@ -135,7 +143,7 @@ public abstract class AttributeKvInsertRepository {
                 int[] result = jdbcTemplate.batchUpdate(BATCH_UPDATE, new BatchPreparedStatementSetter() {
                     @Override
                     public void setValues(PreparedStatement ps, int i) throws SQLException {
-                        ps.setString(1, entities.get(i).getStrValue());
+                        ps.setString(1, replaceNullChars(entities.get(i).getStrValue()));
 
                         if (entities.get(i).getLongValue() != null) {
                             ps.setLong(2, entities.get(i).getLongValue());
@@ -189,8 +197,8 @@ public abstract class AttributeKvInsertRepository {
                         ps.setString(2, insertEntities.get(i).getId().getEntityId());
                         ps.setString(3, insertEntities.get(i).getId().getAttributeType());
                         ps.setString(4, insertEntities.get(i).getId().getAttributeKey());
-                        ps.setString(5, insertEntities.get(i).getStrValue());
-                        ps.setString(10, insertEntities.get(i).getStrValue());
+                        ps.setString(5, replaceNullChars(insertEntities.get(i).getStrValue()));
+                        ps.setString(10, replaceNullChars(insertEntities.get(i).getStrValue()));
 
                         if (insertEntities.get(i).getLongValue() != null) {
                             ps.setLong(6, insertEntities.get(i).getLongValue());
@@ -229,4 +237,10 @@ public abstract class AttributeKvInsertRepository {
         });
     }
 
+    private String replaceNullChars(String strValue) {
+        if (removeNullChars && strValue != null) {
+            return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR);
+        }
+        return strValue;
+    }
 }

dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java

@@ -15,14 +15,19 @@
  */
 package org.thingsboard.server.dao.sqlts;
 
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Repository;
 
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
+import java.util.regex.Pattern;
 
 @Repository
 public abstract class AbstractInsertRepository {
 
+    private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE)));
+    private static final String EMPTY_STR = "";
+
     protected static final String BOOL_V = "bool_v";
     protected static final String STR_V = "str_v";
     protected static final String LONG_V = "long_v";
@@ -46,6 +51,9 @@ public abstract class AbstractInsertRepository {
     protected static final String PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = "str_v = null, bool_v = null, dbl_v = null";
     protected static final String PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, bool_v = null";
 
+    @Value("${sql.remove_null_chars}")
+    private boolean removeNullChars;
+
     @PersistenceContext
     protected EntityManager entityManager;
 
@@ -71,4 +79,11 @@ public abstract class AbstractInsertRepository {
                 throw new RuntimeException("Unsupported insert value: [" + notNullValue + "]");
         }
     }
+
+    protected String replaceNullChars(String strValue) {
+        if (removeNullChars) {
+            return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR);
+        }
+        return strValue;
+    }
 }

dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java

@@ -56,7 +56,7 @@ public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepositor
                 .setParameter("entity_id", entity.getEntityId())
                 .setParameter("key", entity.getKey())
                 .setParameter("ts", entity.getTs())
-                .setParameter("str_v", entity.getStrValue())
+                .setParameter("str_v", replaceNullChars(entity.getStrValue()))
                 .executeUpdate();
     }
 

dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java

@@ -58,7 +58,7 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository {
                 .setParameter("entity_id", entity.getEntityId())
                 .setParameter("key", entity.getKey())
                 .setParameter("ts", entity.getTs())
-                .setParameter("str_v", entity.getStrValue())
+                .setParameter("str_v", replaceNullChars(entity.getStrValue()))
                 .executeUpdate();
     }
 

dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java

@@ -58,7 +58,7 @@ public class PsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepo
                 .setParameter("entity_id", entity.getEntityId())
                 .setParameter("key", entity.getKey())
                 .setParameter("ts", entity.getTs())
-                .setParameter("str_v", entity.getStrValue())
+                .setParameter("str_v", replaceNullChars(entity.getStrValue()))
                 .executeUpdate();
     }