Merge remote-tracking branch 'upstream/rc' into edge-fix-rebalance-too-often

application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java

@@ -106,7 +106,7 @@ public class HousekeeperStatsService {
             this.failedProcessingCounter = register("failedProcessing", statsFactory);
             this.reprocessedCounter = register("reprocessed", statsFactory);
             this.failedReprocessingCounter = register("failedReprocessing", statsFactory);
-            this.processingTimer = statsFactory.createTimer(StatsType.HOUSEKEEPER, "processingTime", "taskType", taskType.name());
+            this.processingTimer = statsFactory.createStatsTimer(StatsType.HOUSEKEEPER.getName(), "processingTime", "taskType", taskType.name());
         }
 
         private StatsCounter register(String statsName, StatsFactory statsFactory) {

application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import jakarta.annotation.PostConstruct;
 import jakarta.annotation.PreDestroy;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.ApplicationEventPublisher;
@@ -116,6 +117,7 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 
 @Service
+@Slf4j
 @TbCoreComponent
 public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCoreNotificationMsg> implements TbCoreConsumerService {
 

application/src/main/resources/thingsboard.yml

@@ -1773,7 +1773,7 @@ queue:
     # EDQS responses topic
     responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}"
     # Poll interval for EDQS topics
-    poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}"
+    poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:25}"
     # Maximum amount of pending requests to EDQS
     max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}"
     # Maximum timeout for requests to EDQS

common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java

@@ -125,7 +125,7 @@ public abstract class BaseEntityData<T extends EntityFields> implements EntityDa
         return switch (key) {
             case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime());
             case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate());
-            case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx));
+            case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx), false);
             default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false);
         };
     }

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java

@@ -17,10 +17,12 @@ package org.thingsboard.server.edqs.data.dp;
 
 import org.thingsboard.server.common.data.kv.DataType;
 
+import java.util.function.Function;
+
 public class CompressedJsonDataPoint extends CompressedStringDataPoint {
 
-    public CompressedJsonDataPoint(long ts, byte[] compressedValue) {
+    public CompressedJsonDataPoint(long ts, byte[] compressedValue, Function<byte[], String> uncompressor) {
-        super(ts, compressedValue);
+        super(ts, compressedValue, uncompressor);
     }
 
     @Override

common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java

@@ -19,17 +19,21 @@ import lombok.Getter;
 import lombok.SneakyThrows;
 import org.thingsboard.common.util.TbBytePool;
 import org.thingsboard.server.common.data.kv.DataType;
-import org.xerial.snappy.Snappy;
+
+import java.util.function.Function;
 
 public class CompressedStringDataPoint extends AbstractDataPoint {
 
     @Getter
     private final byte[] compressedValue;
 
+    protected final Function<byte[], String> uncompressor;
+
     @SneakyThrows
-    public CompressedStringDataPoint(long ts, byte[] compressedValue) {
+    public CompressedStringDataPoint(long ts, byte[] compressedValue, Function<byte[], String> uncompressor) {
         super(ts);
         this.compressedValue = TbBytePool.intern(compressedValue);
+        this.uncompressor = uncompressor;
     }
 
     @Override
@@ -40,7 +44,7 @@ public class CompressedStringDataPoint extends AbstractDataPoint {
     @SneakyThrows
     @Override
     public String getStr() {
-        return Snappy.uncompressString(compressedValue);
+        return uncompressor.apply(compressedValue);
     }
 
     @Override

common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.edqs.repo;
 
 import lombok.AllArgsConstructor;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
 import org.thingsboard.server.common.data.ObjectType;
@@ -40,6 +41,7 @@ import java.util.function.Predicate;
 @Slf4j
 public class DefaultEdqsRepository implements EdqsRepository {
 
+    @Getter
     private final static ConcurrentMap<TenantId, TenantRepo> repos = new ConcurrentHashMap<>();
     private final EdqsStatsService statsService;
 
@@ -52,6 +54,7 @@ public class DefaultEdqsRepository implements EdqsRepository {
         if (event.getEventType() == EdqsEventType.DELETED && event.getObjectType() == ObjectType.TENANT) {
             log.info("Tenant {} deleted", event.getTenantId());
             repos.remove(event.getTenantId());
+            statsService.reportRemoved(ObjectType.TENANT);
         } else {
             get(event.getTenantId()).processEvent(event);
         }
@@ -61,8 +64,7 @@ public class DefaultEdqsRepository implements EdqsRepository {
     public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) {
         long startNs = System.nanoTime();
         long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck);
-        double timingMs = (double) (System.nanoTime() - startNs) / 1000_000;
+        statsService.reportEdqsCountQuery(tenantId, query, System.nanoTime() - startNs);
-        log.info("countEntitiesByQuery done in {} ms", timingMs);
         return result;
     }
 
@@ -71,8 +73,7 @@ public class DefaultEdqsRepository implements EdqsRepository {
                                                        EntityDataQuery query, boolean ignorePermissionCheck) {
         long startNs = System.nanoTime();
         var result = get(tenantId).findEntityDataByQuery(customerId, query, ignorePermissionCheck);
-        double timingMs = (double) (System.nanoTime() - startNs) / 1000_000;
+        statsService.reportEdqsDataQuery(tenantId, query, System.nanoTime() - startNs);
-        log.info("findEntityDataByQuery done in {} ms", timingMs);
         return result;
     }
 

common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java

@@ -332,7 +332,6 @@ public class TenantRepo {
 
     public PageData<QueryResult> findEntityDataByQuery(CustomerId customerId, EntityDataQuery oldQuery, boolean ignorePermissionCheck) {
         EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery);
-        log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query);
         QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck);
         EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query);
         return sortAndConvert(query, queryProcessor.processQuery(), ctx);
@@ -340,7 +339,6 @@ public class TenantRepo {
 
     public long countEntitiesByQuery(CustomerId customerId, EntityCountQuery oldQuery, boolean ignorePermissionCheck) {
         EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery);
-        log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query);
         QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck);
         EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query);
         return queryProcessor.count();

common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java

@@ -15,78 +15,117 @@
  */
 package org.thingsboard.server.edqs.stats;
 
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.TbBytePool;
+import org.thingsboard.common.util.TbStringPool;
 import org.thingsboard.server.common.data.ObjectType;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.query.EntityCountQuery;
 import org.thingsboard.server.common.data.query.EntityDataQuery;
 import org.thingsboard.server.common.stats.EdqsStatsService;
+import org.thingsboard.server.common.stats.StatsCounter;
 import org.thingsboard.server.common.stats.StatsFactory;
 import org.thingsboard.server.common.stats.StatsTimer;
-import org.thingsboard.server.common.stats.StatsType;
+import org.thingsboard.server.edqs.repo.DefaultEdqsRepository;
 
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 @Service
 @Slf4j
+@RequiredArgsConstructor
 @ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && '${queue.edqs.stats.enabled:true}' == 'true'")
 public class DefaultEdqsStatsService implements EdqsStatsService {
 
     private final StatsFactory statsFactory;
 
-    @Value("${queue.edqs.stats.slow_query_threshold:3000}")
+    @Value("${queue.edqs.stats.slow_query_threshold}")
     private int slowQueryThreshold;
 
-    private final ConcurrentHashMap<ObjectType, AtomicInteger> objectCounters = new ConcurrentHashMap<>();
+    private final ConcurrentMap<ObjectType, AtomicInteger> objectCounters = new ConcurrentHashMap<>();
-    private final StatsTimer dataQueryTimer;
+    private final ConcurrentMap<String, StatsTimer> timers = new ConcurrentHashMap<>();
-    private final StatsTimer countQueryTimer;
+    private final ConcurrentMap<String, StatsCounter> counters = new ConcurrentHashMap<>();
 
-    private DefaultEdqsStatsService(StatsFactory statsFactory) {
+    @PostConstruct
-        this.statsFactory = statsFactory;
+    private void init() {
-        dataQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityDataQueryTimer");
+        statsFactory.createGauge("edqsMapGauges", "stringPoolSize", TbStringPool.getPool(), Map::size);
-        countQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityCountQueryTimer");
+        statsFactory.createGauge("edqsMapGauges", "bytePoolSize", TbBytePool.getPool(), Map::size);
+        statsFactory.createGauge("edqsMapGauges", "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size);
     }
 
     @Override
     public void reportAdded(ObjectType objectType) {
-        getObjectCounter(objectType).incrementAndGet();
+        getObjectGauge(objectType).incrementAndGet();
     }
 
     @Override
     public void reportRemoved(ObjectType objectType) {
-        getObjectCounter(objectType).decrementAndGet();
+        getObjectGauge(objectType).decrementAndGet();
     }
 
     @Override
-    public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {
+    public void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {
-        double timingMs = timingNanos / 1000_000.0;
+        checkTiming(tenantId, query, timingNanos);
-        if (timingMs < slowQueryThreshold) {
+        getTimer("entityDataQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS);
-            log.debug("[{}] Executed data query in {} ms: {}", tenantId, timingMs, query);
+    }
-        } else {
+
-            log.warn("[{}] Executed slow data query in {} ms: {}", tenantId, timingMs, query);
+    @Override
-        }
+    public void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {
-        dataQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS);
+        checkTiming(tenantId, query, timingNanos);
+        getTimer("entityCountQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {
+        checkTiming(tenantId, query, timingNanos);
+        getTimer("edqsDataQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {
+        checkTiming(tenantId, query, timingNanos);
+        getTimer("edqsCountQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS);
+    }
+
+    @Override
+    public void reportStringCompressed() {
+        getCounter("stringsCompressed").increment();
     }
 
     @Override
-    public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {
+    public void reportStringUncompressed() {
+        getCounter("stringsUncompressed").increment();
+    }
+
+    private void checkTiming(TenantId tenantId, EntityCountQuery query, long timingNanos) {
         double timingMs = timingNanos / 1000_000.0;
+        String queryType = query instanceof EntityDataQuery ? "data" : "count";
         if (timingMs < slowQueryThreshold) {
-            log.debug("[{}] Executed count query in {} ms: {}", tenantId, timingMs, query);
+            log.debug("[{}] Executed " + queryType + " query in {} ms: {}", tenantId, timingMs, query);
         } else {
-            log.warn("[{}] Executed slow count query in {} ms: {}", tenantId, timingMs, query);
+            log.warn("[{}] Executed slow " + queryType + " query in {} ms: {}", tenantId, timingMs, query);
         }
-        countQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS);
     }
 
-    private AtomicInteger getObjectCounter(ObjectType objectType) {
+    private StatsTimer getTimer(String name) {
+        return timers.computeIfAbsent(name, __ -> statsFactory.createStatsTimer("edqsTimers", name));
+    }
+
+    private StatsCounter getCounter(String name) {
+        return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter("edqsCounters", name));
+    }
+
+    private AtomicInteger getObjectGauge(ObjectType objectType) {
         return objectCounters.computeIfAbsent(objectType, type ->
-                statsFactory.createGauge("edqsObjectsCount", new AtomicInteger(), "objectType", type.name()));
+                statsFactory.createGauge("edqsGauges", "objectsCount", new AtomicInteger(), "objectType", type.name()));
     }
 
 }

common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java

@@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
 import org.thingsboard.server.common.data.kv.KvEntry;
 import org.thingsboard.server.common.data.relation.EntityRelation;
+import org.thingsboard.server.common.stats.EdqsStatsService;
 import org.thingsboard.server.common.util.ProtoUtils;
 import org.thingsboard.server.edqs.data.dp.BoolDataPoint;
 import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint;
@@ -56,14 +57,18 @@ import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto;
 import org.xerial.snappy.Snappy;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 
 @Service
+@RequiredArgsConstructor
 @Slf4j
 public class EdqsConverter {
 
+    private final EdqsStatsService edqsStatsService;
+
     @Value("${queue.edqs.string_compression_length_threshold:512}")
     private int stringCompressionLengthThreshold;
 
@@ -171,26 +176,44 @@ public class EdqsConverter {
         } else if (proto.hasDoubleV()) {
             return new DoubleDataPoint(ts, proto.getDoubleV());
         } else if (proto.hasStringV()) {
-            return new StringDataPoint(ts, proto.getStringV());
+            String stringV = proto.getStringV();
+            if (stringV.length() < stringCompressionLengthThreshold) {
+                return new StringDataPoint(ts, stringV);
+            } else {
+                return new CompressedStringDataPoint(ts, compress(stringV), this::uncompress);
+            }
         } else if (proto.hasCompressedStringV()) {
-            return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray());
+            return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray(), this::uncompress);
         } else if (proto.hasJsonV()) {
-            return new JsonDataPoint(ts, proto.getJsonV());
+            String jsonV = proto.getJsonV();
+            if (jsonV.length() < stringCompressionLengthThreshold) {
+                return new JsonDataPoint(ts, jsonV);
+            } else {
+                return new CompressedJsonDataPoint(ts, compress(jsonV), this::uncompress);
+            }
         } else if (proto.hasCompressedJsonV()) {
-            return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray());
+            return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray(), this::uncompress);
         } else {
             throw new IllegalArgumentException("Unsupported data point proto: " + proto);
         }
     }
 
     @SneakyThrows
-    private static byte[] compress(String value) {
+    private byte[] compress(String value) {
-        byte[] compressed = Snappy.compress(value);
+        byte[] compressed = Snappy.compress(value, StandardCharsets.UTF_8);
-        // TODO: limit the size
+        log.debug("Compressed {} chars to {} bytes", value.length(), compressed.length);
-        log.debug("Compressed {} bytes to {} bytes", value.length(), compressed.length);
+        edqsStatsService.reportStringCompressed();
         return compressed;
     }
 
+    @SneakyThrows
+    private String uncompress(byte[] compressed) {
+        String value = Snappy.uncompressString(compressed, StandardCharsets.UTF_8);
+        log.debug("Uncompressed {} bytes to {} chars", compressed.length, value.length());
+        edqsStatsService.reportStringUncompressed();
+        return value;
+    }
+
     public static Entity toEntity(EntityType entityType, Object entity) {
         Entity edqsEntity = new Entity();
         edqsEntity.setType(entityType);

common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java

@@ -38,7 +38,7 @@ public class EdqsConfig {
     private String requestsTopic;
     @Value("${queue.edqs.responses_topic:edqs.responses}")
     private String responsesTopic;
-    @Value("${queue.edqs.poll_interval:125}")
+    @Value("${queue.edqs.poll_interval:25}")
     private long pollInterval;
     @Value("${queue.edqs.max_pending_requests:10000}")
     private int maxPendingRequests;

common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java

@@ -27,6 +27,7 @@ import org.springframework.stereotype.Service;
 import org.thingsboard.server.common.data.StringUtils;
 
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.ToDoubleFunction;
 
 @Service
 public class DefaultStatsFactory implements StatsFactory {
@@ -86,6 +87,16 @@ public class DefaultStatsFactory implements StatsFactory {
         return meterRegistry.gauge(key, Tags.of(tags), number);
     }
 
+    @Override
+    public <T extends Number> T createGauge(String type, String name, T number, String... tags) {
+        return createGauge(type, number, getTags(name, tags));
+    }
+
+    @Override
+    public <S> void createGauge(String type, String name, S stateObject, ToDoubleFunction<S> numberProvider, String... tags) {
+        meterRegistry.gauge(type, Tags.of(getTags(name, tags)), stateObject, numberProvider);
+    }
+
     @Override
     public MessagesStats createMessagesStats(String key) {
         StatsCounter totalCounter = createStatsCounter(key, TOTAL_MSGS);
@@ -106,8 +117,8 @@ public class DefaultStatsFactory implements StatsFactory {
     }
 
     @Override
-    public StatsTimer createTimer(StatsType type, String name, String... tags) {
+    public StatsTimer createStatsTimer(String type, String name, String... tags) {
-        return new StatsTimer(name, Timer.builder(type.getName())
+        return new StatsTimer(name, Timer.builder(type)
                 .tags(getTags(name, tags))
                 .register(meterRegistry));
     }

common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java

@@ -33,9 +33,21 @@ public class DummyEdqsStatsService implements EdqsStatsService {
     public void reportRemoved(ObjectType objectType) {}
 
     @Override
-    public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {}
+    public void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {}
 
     @Override
-    public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {}
+    public void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {}
+
+    @Override
+    public void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {}
+
+    @Override
+    public void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {}
+
+    @Override
+    public void reportStringCompressed() {}
+
+    @Override
+    public void reportStringUncompressed() {}
 
 }

common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java

@@ -26,8 +26,16 @@ public interface EdqsStatsService {
 
     void reportRemoved(ObjectType objectType);
 
-    void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos);
+    void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos);
 
-    void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos);
+    void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos);
+
+    void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos);
+
+    void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos);
+
+    void reportStringCompressed();
+
+    void reportStringUncompressed();
 
 }

common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java

@@ -17,6 +17,8 @@ package org.thingsboard.server.common.stats;
 
 import io.micrometer.core.instrument.Timer;
 
+import java.util.function.ToDoubleFunction;
+
 public interface StatsFactory {
 
     StatsCounter createStatsCounter(String key, String statsName, String... otherTags);
@@ -25,10 +27,14 @@ public interface StatsFactory {
 
     <T extends Number> T createGauge(String key, T number, String... tags);
 
+    <T extends Number> T createGauge(String type, String name, T number, String... tags);
+
+    <S> void createGauge(String type, String name, S stateObject, ToDoubleFunction<S> numberProvider, String... tags);
+
     MessagesStats createMessagesStats(String key);
 
     Timer createTimer(String key, String... tags);
 
-    StatsTimer createTimer(StatsType type, String name, String... tags);
+    StatsTimer createStatsTimer(String type, String name, String... tags);
 
 }

common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java

@@ -16,12 +16,14 @@
 package org.thingsboard.common.util;
 
 import com.google.common.hash.Hashing;
+import lombok.Getter;
 import org.springframework.util.ConcurrentReferenceHashMap;
 
 import java.util.concurrent.ConcurrentMap;
 
 public class TbBytePool {
 
+    @Getter
     private static final ConcurrentMap<String, byte[]> pool = new ConcurrentReferenceHashMap<>();
 
     public static byte[] intern(byte[] data) {
@@ -32,8 +34,4 @@ public class TbBytePool {
         return pool.computeIfAbsent(checksum, c -> data);
     }
 
-    public static int size(){
-        return pool.size();
-    }
-
 }

common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java

@@ -15,12 +15,14 @@
  */
 package org.thingsboard.common.util;
 
+import lombok.Getter;
 import org.springframework.util.ConcurrentReferenceHashMap;
 
 import java.util.concurrent.ConcurrentMap;
 
 public class TbStringPool {
 
+    @Getter
     private static final ConcurrentMap<String, String> pool = new ConcurrentReferenceHashMap<>();
 
     public static String intern(String data) {
@@ -30,8 +32,4 @@ public class TbStringPool {
         return pool.computeIfAbsent(data, str -> str);
     }
 
-    public static int size(){
-        return pool.size();
-    }
-
 }

dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java

@@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
-import org.thingsboard.common.util.TbStopWatch;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.HasCustomerId;
 import org.thingsboard.server.common.data.HasEmail;
@@ -101,7 +100,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
         validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id);
         validateEntityCountQuery(query);
 
-        TbStopWatch stopWatch = TbStopWatch.create();
+        long startNs = System.nanoTime();
         Long result;
         if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) {
             EdqsRequest request = EdqsRequest.builder()
@@ -112,7 +111,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
         } else {
             result = entityQueryDao.countEntitiesByQuery(tenantId, customerId, query);
         }
-        edqsStatsService.reportCountQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos());
+        edqsStatsService.reportEntityCountQuery(tenantId, query, System.nanoTime() - startNs);
         return result;
     }
 
@@ -123,7 +122,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
         validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id);
         validateEntityDataQuery(query);
 
-        TbStopWatch stopWatch = TbStopWatch.create();
+        long startNs = System.nanoTime();
         PageData<EntityData> result;
         if (edqsApiService.isEnabled() && validForEdqs(query)) {
             EdqsRequest request = EdqsRequest.builder()
@@ -146,7 +145,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
                 }
             }
         }
-        edqsStatsService.reportDataQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos());
+        edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs);
         return result;
     }
 

edqs/src/main/resources/edqs.yml

@@ -66,7 +66,7 @@ queue:
     # EDQS responses topic
     responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}"
     # Poll interval for EDQS topics
-    poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}"
+    poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:25}"
     # Maximum amount of pending requests to EDQS
     max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}"
     # Maximum timeout for requests to EDQS
@@ -76,6 +76,8 @@ queue:
     stats:
       # Enable/disable statistics for EDQS
       enabled: "${TB_EDQS_STATS_ENABLED:true}"
+      # Threshold for slow queries to log, in milliseconds
+      slow_query_threshold: "${TB_EDQS_SLOW_QUERY_THRESHOLD_MS:200}"
 
   kafka:
     # Kafka Bootstrap nodes in "host:port" format

ui-ngx/src/app/modules/common/modules-map.ts

@@ -76,6 +76,8 @@ import * as TranslateCore from '@ngx-translate/core';
 import * as MatDateTimePicker from '@mat-datetimepicker/core';
 import _moment from 'moment';
 import * as tslib from 'tslib';
+import * as Echarts from 'echarts';
+import * as EchartsCore from 'echarts/core';
 
 import * as TbCore from '@core/public-api';
 import * as TbShared from '@shared/public-api';
@@ -405,6 +407,8 @@ class ModulesMap implements IModulesMap {
     '@mat-datetimepicker/core': MatDateTimePicker,
     moment: _moment,
     tslib,
+    'echarts': Echarts,
+    'echarts/core': EchartsCore,
 
     '@core/public-api': TbCore,
     '@shared/public-api': TbShared,

ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html

@@ -33,6 +33,7 @@
                      noMargin
                      isEdit="true"
                      alwaysDisplayTypePrefix
+                     timezone="true"
                      [historyOnly]="onlyHistoryTimewindow"
                      quickIntervalOnly="{{ widgetType === widgetTypes.latest }}"
                      aggregation="{{ widgetType === widgetTypes.timeseries }}"

ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts

@@ -271,7 +271,7 @@ export const updateScadaSymbolMetadataInContent = (svgContent: string, metadata:
   const svgDoc = new DOMParser().parseFromString(svgContent, 'image/svg+xml');
   const parsererror = svgDoc.getElementsByTagName('parsererror');
   if (parsererror?.length) {
-    return parsererror[0].outerHTML;
+    throw Error(parsererror[0].textContent)
   }
   updateScadaSymbolMetadataInDom(svgDoc, metadata);
   return svgDoc.documentElement.outerHTML;

ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html

@@ -80,6 +80,7 @@
                      (fileNameChanged)="entityForm?.get('fileName').patchValue($event)">
       </tb-file-input>
       <tb-js-func *ngIf="entityForm.get('resourceSubType').value === ResourceSubType.MODULE"
+                  [helpPopupStyle]="{ width: '1000px' }"
                   helpId="resource/js-resource-module_fn"
                   formControlName="content"
                   required
@@ -91,8 +92,7 @@
           <label class="tb-title no-padding tb-required"
                  [class.tb-error]="entityForm.get('content').invalid && entityForm.get('content').touched"
                  style="font-size: 16px;">
-            {{ 'javascript.module-script' | translate }}
+            {{ 'javascript.module-script' | translate }}</label>
-          </label>
           <tb-file-input *ngIf="(isAdd || isEdit)"
                          asButton
                          uploadButtonText="{{ 'javascript.upload-from-file' | translate }}"

ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss

@@ -124,7 +124,6 @@
           background: #F3F6FA;
           border-color: #305680;
           padding-right: 38px;
-          overflow: scroll;
           padding-bottom: 4px;
           min-height: 42px;
           scrollbar-width: thin;

ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts

@@ -79,6 +79,7 @@ import {
   SaveWidgetTypeAsDialogResult
 } from '@home/pages/widget/save-widget-type-as-dialog.component';
 import { WidgetService } from '@core/http/widget.service';
+import { ActionNotificationShow } from '@core/notification/notification.actions';
 
 @Component({
   selector: 'tb-scada-symbol',
@@ -213,32 +214,36 @@ export class ScadaSymbolComponent extends PageComponent
         this.editObjectCallbacks.tagsUpdated(tags);
       }
       const metadata: ScadaSymbolMetadata = this.scadaSymbolFormGroup.get('metadata').value;
-      const scadaSymbolContent = this.prepareScadaSymbolContent(metadata);
+      try {
-      const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName,
+        const scadaSymbolContent = this.prepareScadaSymbolContent(metadata);
-        this.symbolData.imageResource.descriptor.mediaType);
+        const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName,
-      const type = imageResourceType(this.symbolData.imageResource);
+          this.symbolData.imageResource.descriptor.mediaType);
-      let imageInfoObservable =
+        const type = imageResourceType(this.symbolData.imageResource);
-        this.imageService.updateImage(type, this.symbolData.imageResource.resourceKey, file);
+        let imageInfoObservable =
-      if (metadata.title !== this.symbolData.imageResource.title) {
+          this.imageService.updateImage(type, this.symbolData.imageResource.resourceKey, file);
-        imageInfoObservable = imageInfoObservable.pipe(
+        if (metadata.title !== this.symbolData.imageResource.title) {
-          switchMap(imageInfo => {
+          imageInfoObservable = imageInfoObservable.pipe(
-            imageInfo.title = metadata.title;
+            switchMap(imageInfo => {
-            return this.imageService.updateImageInfo(imageInfo);
+              imageInfo.title = metadata.title;
-          })
+              return this.imageService.updateImageInfo(imageInfo);
-        );
+            })
-      }
+          );
-      imageInfoObservable.pipe(
+        }
-        switchMap(imageInfo => this.imageService.getImageString(
+        imageInfoObservable.pipe(
+          switchMap(imageInfo => this.imageService.getImageString(
             `${IMAGES_URL_PREFIX}/${type}/${encodeURIComponent(imageInfo.resourceKey)}`).pipe(
-              map(content => ({
+            map(content => ({
-                imageResource: imageInfo,
+              imageResource: imageInfo,
-                scadaSymbolContent: content
+              scadaSymbolContent: content
-              }))
+            }))
           ))
-      ).subscribe(data => {
+        ).subscribe(data => {
-        this.init(data);
+          this.init(data);
-        this.updateBreadcrumbs.emit();
+          this.updateBreadcrumbs.emit();
-      });
+        });
+      } catch (e) {
+        this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' }));
+      }
     }
   }
 
@@ -248,43 +253,47 @@ export class ScadaSymbolComponent extends PageComponent
 
   enterPreviewMode() {
     this.previewMetadata = this.scadaSymbolFormGroup.get('metadata').value;
-    this.symbolData.scadaSymbolContent = this.prepareScadaSymbolContent(this.previewMetadata);
+    try {
-    this.previewScadaSymbolObjectSettings = {
+      this.symbolData.scadaSymbolContent = this.prepareScadaSymbolContent(this.previewMetadata);
-      behavior: {},
+      this.previewScadaSymbolObjectSettings = {
-      properties: {}
+        behavior: {},
-    };
+        properties: {}
-    this.scadaPreviewFormGroup.patchValue({
+      };
-      scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings
+      this.scadaPreviewFormGroup.patchValue({
-    }, {emitEvent: false});
+        scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings
-    this.scadaPreviewFormGroup.markAsPristine();
+      }, {emitEvent: false});
-    const settings: ScadaSymbolWidgetSettings = {...scadaSymbolWidgetDefaultSettings,
+      this.scadaPreviewFormGroup.markAsPristine();
-      ...{
+      const settings: ScadaSymbolWidgetSettings = {...scadaSymbolWidgetDefaultSettings,
+        ...{
           simulated: true,
           scadaSymbolUrl: null,
           scadaSymbolContent: this.symbolData.scadaSymbolContent,
           scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings,
           padding: '0',
           background: colorBackground('rgba(0,0,0,0)')
-         }
+        }
-    };
+      };
-    this.previewWidget = {
+      this.previewWidget = {
-      typeFullFqn: 'system.scada_symbol',
+        typeFullFqn: 'system.scada_symbol',
-      type: widgetType.rpc,
+        type: widgetType.rpc,
-      sizeX: this.previewMetadata.widgetSizeX || 3,
+        sizeX: this.previewMetadata.widgetSizeX || 3,
-      sizeY: this.previewMetadata.widgetSizeY || 3,
+        sizeY: this.previewMetadata.widgetSizeY || 3,
-      row: 0,
+        row: 0,
-      col: 0,
+        col: 0,
-      config: {
+        config: {
-        settings,
+          settings,
-        showTitle: false,
+          showTitle: false,
-        dropShadow: false,
+          dropShadow: false,
-        padding: '0',
+          padding: '0',
-        margin: '0',
+          margin: '0',
-        backgroundColor: 'rgba(0,0,0,0)'
+          backgroundColor: 'rgba(0,0,0,0)'
-      }
+        }
-    };
+      };
-    this.previewWidgets = [this.previewWidget];
+      this.previewWidgets = [this.previewWidget];
-    this.previewMode = true;
+      this.previewMode = true;
+    } catch (e) {
+      this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' }));
+    }
   }
 
   exitPreviewMode() {
@@ -374,19 +383,23 @@ export class ScadaSymbolComponent extends PageComponent
       metadata = parseScadaSymbolMetadataFromContent(this.origSymbolData.scadaSymbolContent);
     }
     const linkElement = document.createElement('a');
-    const scadaSymbolContent = this.prepareScadaSymbolContent(metadata);
+    try {
-    const blob = new Blob([scadaSymbolContent], { type: this.symbolData.imageResource.descriptor.mediaType });
+      const scadaSymbolContent = this.prepareScadaSymbolContent(metadata);
-    const url = URL.createObjectURL(blob);
+      const blob = new Blob([scadaSymbolContent], { type: this.symbolData.imageResource.descriptor.mediaType });
-    linkElement.setAttribute('href', url);
+      const url = URL.createObjectURL(blob);
-    linkElement.setAttribute('download', this.symbolData.imageResource.fileName);
+      linkElement.setAttribute('href', url);
-    const clickEvent = new MouseEvent('click',
+      linkElement.setAttribute('download', this.symbolData.imageResource.fileName);
-      {
+      const clickEvent = new MouseEvent('click',
-        view: window,
+        {
-        bubbles: true,
+          view: window,
-        cancelable: false
+          bubbles: true,
-      }
+          cancelable: false
-    );
+        }
-    linkElement.dispatchEvent(clickEvent);
+      );
+      linkElement.dispatchEvent(clickEvent);
+    } catch (e) {
+      this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' }));
+    }
   }
 
   createWidget() {

ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html

@@ -28,8 +28,8 @@
   <mat-progress-bar color="warn" mode="indeterminate" *ngIf="isLoading$ | async">
   </mat-progress-bar>
   <div style="height: 4px;" *ngIf="!(isLoading$ | async)"></div>
-  <div mat-dialog-content>
+  <div mat-dialog-content class="relative">
-    <fieldset [disabled]="isLoading$ | async">
+    <fieldset [disabled]="isLoading$ | async" tb-toast toastTarget="uploadRoot" >
       <tb-image-input resultAsFile
                       processImageApiLink
                       label="{{(isScada ? 'scada.symbol-preview' : 'image.image-preview') | translate}}"

ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts

@@ -41,6 +41,7 @@ import {
   updateScadaSymbolMetadataInContent
 } from '@home/components/widget/lib/scada/scada-symbol.models';
 import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
+import { ActionNotificationShow } from '@core/notification/notification.actions';
 
 export interface UploadImageDialogData {
   imageSubType: ResourceSubType;
@@ -135,38 +136,48 @@ export class UploadImageDialogComponent extends
   upload(): void {
     this.submitted = true;
     let file: File = this.uploadImageFormGroup.get('file').value;
-    if (this.uploadImage) {
+    try {
-      const title: string = this.uploadImageFormGroup.get('title').value;
+      if (this.uploadImage) {
-      if (this.isScada) {
+        const title: string = this.uploadImageFormGroup.get('title').value;
-        if (!this.scadaSymbolMetadata) {
+        if (this.isScada) {
-          this.scadaSymbolMetadata = emptyMetadata();
+          if (!this.scadaSymbolMetadata) {
-        }
+            this.scadaSymbolMetadata = emptyMetadata();
-        if (this.scadaSymbolMetadata.title !== title) {
+          }
-          this.scadaSymbolMetadata.title = title;
+          if (this.scadaSymbolMetadata.title !== title) {
+            this.scadaSymbolMetadata.title = title;
+          }
+          const newContent = updateScadaSymbolMetadataInContent(this.scadaSymbolContent, this.scadaSymbolMetadata);
+          file = updateFileContent(file, newContent);
         }
-        const newContent = updateScadaSymbolMetadataInContent(this.scadaSymbolContent, this.scadaSymbolMetadata);
-        file = updateFileContent(file, newContent);
-      }
-      forkJoin([
-        this.imageService.uploadImage(file, title, this.data.imageSubType),
-        blobToBase64(file)
-      ]).subscribe(([imageInfo, base64]) => {
-        this.dialogRef.close({image: Object.assign(imageInfo, {base64})});
-      });
-    } else {
-      if (this.isScada) {
-        blobToText(file).subscribe(scadaSymbolContent => {
-          this.dialogRef.close({scadaSymbolContent});
-        });
-      } else {
-        const image = this.data.image;
         forkJoin([
-          this.imageService.updateImage(imageResourceType(image), image.resourceKey, file),
+          this.imageService.uploadImage(file, title, this.data.imageSubType),
           blobToBase64(file)
         ]).subscribe(([imageInfo, base64]) => {
-          this.dialogRef.close({image:Object.assign(imageInfo, {base64})});
+          this.dialogRef.close({image: Object.assign(imageInfo, {base64})});
         });
+      } else {
+        if (this.isScada) {
+          blobToText(file).subscribe(scadaSymbolContent => {
+            this.dialogRef.close({scadaSymbolContent});
+          });
+        } else {
+          const image = this.data.image;
+          forkJoin([
+            this.imageService.updateImage(imageResourceType(image), image.resourceKey, file),
+            blobToBase64(file)
+          ]).subscribe(([imageInfo, base64]) => {
+            this.dialogRef.close({image:Object.assign(imageInfo, {base64})});
+          });
+        }
       }
+    } catch (e) {
+      this.store.dispatch(new ActionNotificationShow({
+        message: e.message,
+        type: 'error',
+        verticalPosition: 'top',
+        horizontalPosition: 'right',
+        target: 'uploadRoot'
+      }));
     }
   }
 }

ui-ngx/src/app/shared/components/js-func.component.scss

@@ -85,4 +85,8 @@
       color: rgb(49, 132, 149);
     }
   }
+
+  label.tb-title.tb-required::after {
+    content: "*";
+  }
 }

ui-ngx/src/app/shared/components/markdown.component.scss

@@ -270,7 +270,7 @@
         outline: none;
         position: absolute;
         width: 206px;
-        height: 42px;
+        height: 32px;
         top: 0;
         right: 32px;
         background: 0 0;
@@ -281,11 +281,11 @@
         user-select: none;
 
         &.multiline {
-          right: 44px;
+          right: 38px;
         }
 
         p {
-          padding: 8px;
+          padding: 8px 8px 0;
           top: 1px;
           transition: .2s;
           color: #2a7dec;
@@ -301,10 +301,10 @@
           background-color: #fff;
           position: absolute;
           width: 38px;
-          height: 38px;
+          height: 28px;
           top: 3px;
           right: 3px;
-          padding: 10px;
+          padding: 10px 10px 0;
 
           img {
             position: initial;