Merge pull request #8472 from YevhenBondarenko/kv-validator

kv validator improvements

application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@@ -43,6 +43,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback;
 import org.thingsboard.server.common.stats.TbApiUsageReportClient;
 import org.thingsboard.server.dao.attributes.AttributesService;
 import org.thingsboard.server.dao.timeseries.TimeseriesService;
+import org.thingsboard.server.dao.util.KvUtils;
 import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
 import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService;
 import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
@@ -134,6 +135,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
         checkInternalEntity(entityId);
         boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null;
         if (sysTenant || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
+            KvUtils.validate(ts);
             if (saveLatest) {
                 saveAndNotifyInternal(tenantId, entityId, ts, ttl, getCallback(tenantId, customerId, sysTenant, callback));
             } else {

dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java

@@ -30,21 +30,18 @@ import java.util.Optional;
 @Slf4j
 public class NoXssValidator implements ConstraintValidator<NoXss, Object> {
     private static final AntiSamy xssChecker = new AntiSamy();
-    private static Policy xssPolicy;
+    private static final Policy xssPolicy;
 
-    @Override
-    public void initialize(NoXss constraintAnnotation) {
-        if (xssPolicy == null) {
-            xssPolicy = Optional.ofNullable(getClass().getClassLoader().getResourceAsStream("xss-policy.xml"))
-                    .map(inputStream -> {
-                        try {
-                            return Policy.getInstance(inputStream);
-                        } catch (Exception e) {
-                            throw new RuntimeException(e);
-                        }
-                    })
-                    .orElseThrow(() -> new IllegalStateException("XSS policy file not found"));
-        }
+    static {
+        xssPolicy = Optional.ofNullable(NoXssValidator.class.getClassLoader().getResourceAsStream("xss-policy.xml"))
+                .map(inputStream -> {
+                    try {
+                        return Policy.getInstance(inputStream);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                })
+                .orElseThrow(() -> new IllegalStateException("XSS policy file not found"));
     }
 
     @Override
@@ -55,10 +52,13 @@ public class NoXssValidator implements ConstraintValidator<NoXss, Object> {
         } else {
             return true;
         }
+        return isValid(stringValue);
+    }
+
+    public static boolean isValid(String stringValue) {
         if (stringValue.isEmpty()) {
             return true;
         }
-
         try {
             return xssChecker.scan(stringValue, xssPolicy).getNumberOfErrors() == 0;
         } catch (ScanException | PolicyException e) {

dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java

@@ -42,7 +42,6 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult;
 import org.thingsboard.server.dao.entityview.EntityViewService;
 import org.thingsboard.server.dao.exception.IncorrectParameterException;
 import org.thingsboard.server.dao.service.Validator;
-import org.thingsboard.server.dao.util.KvUtils;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -156,7 +155,6 @@ public class BaseTimeseriesService implements TimeseriesService {
 
     @Override
     public ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
-        KvUtils.validate(tsKvEntry);
         validate(entityId);
         List<ListenableFuture<Integer>> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY);
         saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L);
@@ -174,7 +172,6 @@ public class BaseTimeseriesService implements TimeseriesService {
     }
 
     private ListenableFuture<Integer> doSave(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntries, long ttl, boolean saveLatest) {
-        KvUtils.validate(tsKvEntries);
         int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST;
         List<ListenableFuture<Integer>> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * inserts);
         for (TsKvEntry tsKvEntry : tsKvEntries) {
@@ -189,7 +186,6 @@ public class BaseTimeseriesService implements TimeseriesService {
 
     @Override
     public ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntries) {
-        KvUtils.validate(tsKvEntries);
         List<ListenableFuture<Void>> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size());
         for (TsKvEntry tsKvEntry : tsKvEntries) {
             futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry));

dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java

@@ -15,13 +15,27 @@
  */
 package org.thingsboard.server.dao.util;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.dao.exception.DataValidationException;
 import org.thingsboard.server.dao.exception.IncorrectParameterException;
-import org.thingsboard.server.dao.service.ConstraintValidator;
+import org.thingsboard.server.dao.service.NoXssValidator;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 public class KvUtils {
+
+    private static final Cache<String, Boolean> validatedKeys;
+
+    static {
+        validatedKeys = Caffeine.newBuilder()
+                .weakKeys()
+                .expireAfterAccess(60, TimeUnit.MINUTES)
+                .maximumSize(100000).build();
+    }
+
     public static void validate(List<? extends KvEntry> tsKvEntries) {
         tsKvEntries.forEach(KvUtils::validate);
     }
@@ -30,6 +44,24 @@ public class KvUtils {
         if (tsKvEntry == null) {
             throw new IncorrectParameterException("Key value entry can't be null");
         }
-        ConstraintValidator.validateFields(tsKvEntry);
+
+        String key = tsKvEntry.getKey();
+
+        if (key == null) {
+            throw new DataValidationException("Key can't be null");
+        }
+
+        if (validatedKeys.getIfPresent(key) != null) {
+            return;
+        }
+
+        if (!NoXssValidator.isValid(key)) {
+            throw new DataValidationException("Validation error: key is malformed");
+        }
+        if (key.length() > 255) {
+            throw new DataValidationException("Validation error: key length must be equal or less than 255");
+        }
+
+        validatedKeys.put(key, Boolean.TRUE);
     }
 }