From 64a3baad4109909fdf2e6651f208d5505827be10 Mon Sep 17 00:00:00 2001 From: Dima Landiak Date: Wed, 27 Nov 2019 16:58:48 +0200 Subject: [PATCH] remove null chars from str value of attributes and telemetry for sql --- .../src/main/resources/thingsboard.yml | 2 ++ .../AttributeKvInsertRepository.java | 22 +++++++++++++++---- .../dao/sqlts/AbstractInsertRepository.java | 15 +++++++++++++ .../timescale/TimescaleInsertRepository.java | 2 +- .../sqlts/ts/PsqlLatestInsertRepository.java | 2 +- .../ts/PsqlTimeseriesInsertRepository.java | 2 +- 6 files changed, 38 insertions(+), 7 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 764516b9c1..8fd418a153 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -204,6 +204,8 @@ sql: batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:1000}" + # Specify whether to remove null characters from strValue of attributes and timeseries before insert + remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" # Actor system parameters actors: diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 89843048a0..0a537cbe01 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.attributes; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.jpa.repository.Modifying; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; @@ -34,12 +35,16 @@ import java.sql.SQLException; import java.sql.Types; import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; @SqlDao @Repository @Slf4j public abstract class AttributeKvInsertRepository { + private static final ThreadLocal PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); + private static final String EMPTY_STR = ""; + private static final String BATCH_UPDATE = "UPDATE attribute_kv SET str_v = ?, long_v = ?, dbl_v = ?, bool_v = ?, last_update_ts = ? " + "WHERE entity_type = ? and entity_id = ? and attribute_type =? and attribute_key = ?;"; @@ -60,6 +65,9 @@ public abstract class AttributeKvInsertRepository { @Autowired private TransactionTemplate transactionTemplate; + @Value("${sql.remove_null_chars}") + private boolean removeNullChars; + @PersistenceContext protected EntityManager entityManager; @@ -99,7 +107,7 @@ public abstract class AttributeKvInsertRepository { .setParameter("entity_id", entity.getId().getEntityId()) .setParameter("attribute_type", entity.getId().getAttributeType()) .setParameter("attribute_key", entity.getId().getAttributeKey()) - .setParameter("str_v", entity.getStrValue()) + .setParameter("str_v", replaceNullChars(entity.getStrValue())) .setParameter("last_update_ts", entity.getLastUpdateTs()) .executeUpdate(); } @@ -135,7 +143,7 @@ public abstract class AttributeKvInsertRepository { int[] result = jdbcTemplate.batchUpdate(BATCH_UPDATE, new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { - ps.setString(1, entities.get(i).getStrValue()); + ps.setString(1, replaceNullChars(entities.get(i).getStrValue())); if (entities.get(i).getLongValue() != null) { ps.setLong(2, entities.get(i).getLongValue()); @@ -189,8 +197,8 @@ public abstract class AttributeKvInsertRepository { ps.setString(2, insertEntities.get(i).getId().getEntityId()); ps.setString(3, insertEntities.get(i).getId().getAttributeType()); ps.setString(4, insertEntities.get(i).getId().getAttributeKey()); - ps.setString(5, insertEntities.get(i).getStrValue()); - ps.setString(10, insertEntities.get(i).getStrValue()); + ps.setString(5, replaceNullChars(insertEntities.get(i).getStrValue())); + ps.setString(10, replaceNullChars(insertEntities.get(i).getStrValue())); if (insertEntities.get(i).getLongValue() != null) { ps.setLong(6, insertEntities.get(i).getLongValue()); @@ -229,4 +237,10 @@ public abstract class AttributeKvInsertRepository { }); } + private String replaceNullChars(String strValue) { + if (removeNullChars && strValue != null) { + return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); + } + return strValue; + } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java index 274b07e4fc..919ab5314d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractInsertRepository.java @@ -15,14 +15,19 @@ */ package org.thingsboard.server.dao.sqlts; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; +import java.util.regex.Pattern; @Repository public abstract class AbstractInsertRepository { + private static final ThreadLocal PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); + private static final String EMPTY_STR = ""; + protected static final String BOOL_V = "bool_v"; protected static final String STR_V = "str_v"; protected static final String LONG_V = "long_v"; @@ -46,6 +51,9 @@ public abstract class AbstractInsertRepository { protected static final String PSQL_ON_LONG_VALUE_UPDATE_SET_NULLS = "str_v = null, bool_v = null, dbl_v = null"; protected static final String PSQL_ON_DBL_VALUE_UPDATE_SET_NULLS = "str_v = null, long_v = null, bool_v = null"; + @Value("${sql.remove_null_chars}") + private boolean removeNullChars; + @PersistenceContext protected EntityManager entityManager; @@ -71,4 +79,11 @@ public abstract class AbstractInsertRepository { throw new RuntimeException("Unsupported insert value: [" + notNullValue + "]"); } } + + protected String replaceNullChars(String strValue) { + if (removeNullChars) { + return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); + } + return strValue; + } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java index d4cbd1c994..11f4ea4b5d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleInsertRepository.java @@ -56,7 +56,7 @@ public class TimescaleInsertRepository extends AbstractTimeseriesInsertRepositor .setParameter("entity_id", entity.getEntityId()) .setParameter("key", entity.getKey()) .setParameter("ts", entity.getTs()) - .setParameter("str_v", entity.getStrValue()) + .setParameter("str_v", replaceNullChars(entity.getStrValue())) .executeUpdate(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java index c61a74a15d..5d50bf0dd9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlLatestInsertRepository.java @@ -58,7 +58,7 @@ public class PsqlLatestInsertRepository extends AbstractLatestInsertRepository { .setParameter("entity_id", entity.getEntityId()) .setParameter("key", entity.getKey()) .setParameter("ts", entity.getTs()) - .setParameter("str_v", entity.getStrValue()) + .setParameter("str_v", replaceNullChars(entity.getStrValue())) .executeUpdate(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java index 6390a7faee..0baea27d7b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/PsqlTimeseriesInsertRepository.java @@ -58,7 +58,7 @@ public class PsqlTimeseriesInsertRepository extends AbstractTimeseriesInsertRepo .setParameter("entity_id", entity.getEntityId()) .setParameter("key", entity.getKey()) .setParameter("ts", entity.getTs()) - .setParameter("str_v", entity.getStrValue()) + .setParameter("str_v", replaceNullChars(entity.getStrValue())) .executeUpdate(); }