Merge pull request #7541 from ViacheslavKlimov/fix/ts-sort-nulls-last

[3.4.2] Use default SQL null ordering for ts_kv

application/src/main/resources/thingsboard.yml

@@ -534,6 +534,7 @@ spring.servlet.multipart.max-file-size: "50MB"
 spring.servlet.multipart.max-request-size: "50MB"
 
 spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true"
+# Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this parameter is used
 spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}"
 
 # SQL DAO Configuration

dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java

@@ -20,7 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.PageRequest;
-import org.springframework.data.domain.Sort;
+import org.springframework.data.domain.Sort.Direction;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.kv.Aggregation;
@@ -143,8 +143,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
                 keyId,
                 query.getStartTs(),
                 query.getEndTs(),
-                PageRequest.of(0, query.getLimit(),
+                PageRequest.ofSize(query.getLimit()).withSort(Direction.fromString(query.getOrder()), "ts"));
-                        Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative())));
         tsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(query.getKey()));
         List<TsKvEntry> tsKvEntries = DaoUtil.convertDataList(tsKvEntities);
         long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs());

dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java

@@ -175,9 +175,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
                 keyId,
                 query.getStartTs(),
                 query.getEndTs(),
-                PageRequest.of(0, query.getLimit(),
+                PageRequest.ofSize(query.getLimit()).withSort(Sort.Direction.fromString(query.getOrder()), "ts"));
-                        Sort.by(new Sort.Order(Sort.Direction.fromString(query.getOrder()), "ts").nullsNative())));
-        ;
         timescaleTsKvEntities.forEach(tsKvEntity -> tsKvEntity.setStrKey(strKey));
         var tsKvEntries = DaoUtil.convertDataList(timescaleTsKvEntities);
         long lastTs = tsKvEntries.stream().map(TsKvEntry::getTs).max(Long::compare).orElse(query.getStartTs());

dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java

@@ -31,14 +31,13 @@ import java.util.UUID;
 @TimescaleDBTsOrTsLatestDao
 public interface TsKvTimescaleRepository extends JpaRepository<TimescaleTsKvEntity, TimescaleTsKvCompositeKey> {
 
-    @Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " +
+    @Query(value = "SELECT * FROM ts_kv WHERE entity_id = :entityId " +
-            "AND tskv.key = :entityKey " +
+            "AND key = :entityKey AND ts >= :startTs AND ts < :endTs", nativeQuery = true)
-            "AND tskv.ts >= :startTs AND tskv.ts < :endTs")
+    List<TimescaleTsKvEntity> findAllWithLimit(@Param("entityId") UUID entityId,
-    List<TimescaleTsKvEntity> findAllWithLimit(
+                                               @Param("entityKey") int key,
-            @Param("entityId") UUID entityId,
+                                               @Param("startTs") long startTs,
-            @Param("entityKey") int key,
+                                               @Param("endTs") long endTs,
-            @Param("startTs") long startTs,
+                                               Pageable pageable);
-            @Param("endTs") long endTs, Pageable pageable);
 
     @Transactional
     @Modifying

dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java

@@ -29,8 +29,15 @@ import java.util.UUID;
 
 public interface TsKvRepository extends JpaRepository<TsKvEntity, TsKvCompositeKey> {
 
-    @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " +
+    /*
-            "AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs")
+    * Using native query to avoid adding 'nulls first' or 'nulls last' (ignoring spring.jpa.properties.hibernate.order_by.default_null_ordering)
+    * to the order so that index scan is done instead of full scan.
+    *
+    * Note: even when setting custom NullHandling for the Sort.Order for non-native queries,
+    * it will be ignored and default_null_ordering will be used
+    * */
+    @Query(value = "SELECT * FROM ts_kv WHERE entity_id = :entityId " +
+            "AND key = :entityKey AND ts >= :startTs AND ts < :endTs ", nativeQuery = true)
     List<TsKvEntity> findAllWithLimit(@Param("entityId") UUID entityId,
                                       @Param("entityKey") int key,
                                       @Param("startTs") long startTs,