@ -17,6 +17,9 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.type.TypeReference ;
import com.fasterxml.jackson.databind.JsonNode ;
import com.fasterxml.jackson.databind.node.BooleanNode ;
import com.fasterxml.jackson.databind.node.DoubleNode ;
import com.fasterxml.jackson.databind.node.IntNode ;
import com.fasterxml.jackson.databind.node.ObjectNode ;
import org.junit.After ;
import org.junit.Assert ;
@ -43,12 +46,21 @@ import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.asset.Asset ;
import org.thingsboard.server.common.data.id.DeviceId ;
import org.thingsboard.server.common.data.id.EntityId ;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry ;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry ;
import org.thingsboard.server.common.data.kv.BooleanDataEntry ;
import org.thingsboard.server.common.data.kv.DoubleDataEntry ;
import org.thingsboard.server.common.data.kv.JsonDataEntry ;
import org.thingsboard.server.common.data.kv.LongDataEntry ;
import org.thingsboard.server.common.data.kv.StringDataEntry ;
import org.thingsboard.server.common.data.page.PageData ;
import org.thingsboard.server.common.data.query.AlarmCountQuery ;
import org.thingsboard.server.common.data.query.AlarmData ;
import org.thingsboard.server.common.data.query.AlarmDataPageLink ;
import org.thingsboard.server.common.data.query.AlarmDataQuery ;
import org.thingsboard.server.common.data.query.AliasEntityId ;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2 ;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2.KeyInfo ;
import org.thingsboard.server.common.data.query.DeviceTypeFilter ;
import org.thingsboard.server.common.data.query.DynamicValue ;
import org.thingsboard.server.common.data.query.DynamicValueSourceType ;
@ -84,6 +96,7 @@ import java.util.ArrayList;
import java.util.Arrays ;
import java.util.Collections ;
import java.util.List ;
import java.util.UUID ;
import java.util.concurrent.TimeUnit ;
import java.util.function.BiConsumer ;
import java.util.stream.Collectors ;
@ -1329,7 +1342,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
//assign dashboard
doPost ( "/api/customer/" + savedCustomer . getId ( ) . getId ( ) . toString ( )
+ "/dashboard/" + savedDashboard . getId ( ) . getId ( ) . toString ( ) , Dashboard . class ) ;
+ "/dashboard/" + savedDashboard . getId ( ) . getId ( ) . toString ( ) , Dashboard . class ) ;
// check entity data query by customer
User customerUser = new User ( ) ;
@ -1494,4 +1507,213 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
return nameFilter ;
}
// --- findAvailableEntityKeysV2 tests ---
@Test
public void testFindAvailableKeysByQueryV2 ( ) throws Exception {
// GIVEN — two devices matched by query; a third device should not be matched
var device1 = createDevice ( "Test device 1" ) ;
var device2 = createDevice ( "Test device 2" ) ;
var unmatchedDevice = createDevice ( "Unmatched device" ) ;
// unmatched device has unique keys that must NOT appear in the result
postTelemetry ( unmatchedDevice . getId ( ) , new BasicTsKvEntry ( 9000 , new DoubleDataEntry ( "unmatchedTs" , 999 . 0 ) ) ) ;
postAttributes ( unmatchedDevice . getId ( ) , AttributeScope . SHARED_SCOPE , new StringDataEntry ( "unmatchedAttr" , "nope" ) ) ;
// device1: timeseries1 (Double) with two data points, and timeseries2 older data point
postTelemetry ( device1 . getId ( ) , new BasicTsKvEntry ( 1000 , new DoubleDataEntry ( "timeseries1" , 10 . 0 ) ) ) ;
postTelemetry ( device1 . getId ( ) , new BasicTsKvEntry ( 2000 , new DoubleDataEntry ( "timeseries1" , 20 . 5 ) ) ) ;
postTelemetry ( device1 . getId ( ) , new BasicTsKvEntry ( 1000 , new LongDataEntry ( "timeseries2" , 100L ) ) ) ;
// device2: timeseries2 (Long) with a newer data point, and timeseries3 only on this device
postTelemetry ( device2 . getId ( ) , new BasicTsKvEntry ( 3000 , new LongDataEntry ( "timeseries2" , 300L ) ) ) ;
postTelemetry ( device2 . getId ( ) , new BasicTsKvEntry ( 5000 , new DoubleDataEntry ( "timeseries3" , 99 . 9 ) ) ) ;
// device1: SHARED_SCOPE attributes
postAttributes ( device1 . getId ( ) , AttributeScope . SHARED_SCOPE ,
new BooleanDataEntry ( "sharedAttribute1" , true ) , new DoubleDataEntry ( "sharedAttribute2" , 3 . 14 ) ) ;
// device2: CLIENT_SCOPE attributes (saved via service to bypass API restriction)
attributesService . save ( tenantId , device2 . getId ( ) , AttributeScope . CLIENT_SCOPE , List . of (
new BaseAttributeKvEntry ( new JsonDataEntry ( "clientAttribute1" , "{\"key\":\"val\"}" ) , System . currentTimeMillis ( ) ) ,
new BaseAttributeKvEntry ( new BooleanDataEntry ( "clientAttribute2" , false ) , System . currentTimeMillis ( ) )
) ) . get ( ) ;
// device1 also has SERVER_SCOPE attributes (should be omitted by scope filter)
postAttributes ( device1 . getId ( ) , AttributeScope . SERVER_SCOPE ,
new StringDataEntry ( "serverAttribute1" , "sv1" ) , new LongDataEntry ( "serverAttribute2" , 42L ) ) ;
// WHEN — query matches both devices; request timeseries + only SHARED and CLIENT attribute scopes
DeviceTypeFilter filter = new DeviceTypeFilter ( ) ;
filter . setDeviceTypes ( List . of ( "default" ) ) ;
filter . setDeviceNameFilter ( "Test device" ) ;
EntityDataPageLink pageLink = new EntityDataPageLink ( 100 , 0 , null , null ) ;
EntityDataQuery query = new EntityDataQuery ( filter , pageLink , List . of ( ) , null , null ) ;
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2 ( query ,
true , true , List . of ( AttributeScope . SHARED_SCOPE , AttributeScope . CLIENT_SCOPE ) , true ) ;
// THEN
assertThat ( result . entityTypes ( ) ) . containsExactly ( EntityType . DEVICE ) ;
// timeseries: keys collected from both devices, samples contain the freshest data points
assertThat ( result . timeseries ( ) ) . extracting ( KeyInfo : : key )
. containsExactly ( "timeseries1" , "timeseries2" , "timeseries3" ) ;
assertThat ( result . timeseries ( ) ) . allSatisfy ( ki - > assertThat ( ki . sample ( ) ) . isNotNull ( ) ) ;
assertKeySample ( result . timeseries ( ) , "timeseries1" , new DoubleNode ( 20 . 5 ) , 2000 ) ; // from device1
assertKeySample ( result . timeseries ( ) , "timeseries2" , new IntNode ( 300 ) , 3000 ) ; // from device2 (newer)
assertKeySample ( result . timeseries ( ) , "timeseries3" , new DoubleNode ( 99 . 9 ) , 5000 ) ; // only on device2
// SERVER_SCOPE must be fully omitted from the response
assertThat ( result . attributes ( ) ) . containsOnlyKeys ( AttributeScope . SHARED_SCOPE , AttributeScope . CLIENT_SCOPE ) ;
// SHARED_SCOPE: from device1 (alphabetical order)
assertThat ( result . attributes ( ) . get ( AttributeScope . SHARED_SCOPE ) )
. extracting ( KeyInfo : : key ) . containsExactly ( "sharedAttribute1" , "sharedAttribute2" ) ;
assertKeySample ( result . attributes ( ) . get ( AttributeScope . SHARED_SCOPE ) , "sharedAttribute1" , BooleanNode . TRUE ) ;
assertKeySample ( result . attributes ( ) . get ( AttributeScope . SHARED_SCOPE ) , "sharedAttribute2" , new DoubleNode ( 3 . 14 ) ) ;
// CLIENT_SCOPE: from device2 (alphabetical order)
assertThat ( result . attributes ( ) . get ( AttributeScope . CLIENT_SCOPE ) )
. extracting ( KeyInfo : : key ) . containsExactly ( "clientAttribute1" , "clientAttribute2" ) ;
assertKeySample ( result . attributes ( ) . get ( AttributeScope . CLIENT_SCOPE ) , "clientAttribute1" , JacksonUtil . toJsonNode ( "{\"key\":\"val\"}" ) ) ;
assertKeySample ( result . attributes ( ) . get ( AttributeScope . CLIENT_SCOPE ) , "clientAttribute2" , BooleanNode . FALSE ) ;
}
@Test
public void testFindAvailableKeysByQueryV2_withoutSamples ( ) throws Exception {
// GIVEN
var device = createDevice ( "Test device" ) ;
postTelemetry ( device . getId ( ) , new BasicTsKvEntry ( System . currentTimeMillis ( ) , new DoubleDataEntry ( "temperature" , 10 . 0 ) ) ) ;
postAttributes ( device . getId ( ) , AttributeScope . SERVER_SCOPE , new StringDataEntry ( "firmware" , "v1.0" ) ) ;
// WHEN
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2 (
buildDeviceQuery ( "Test device" ) , true , true , null , false ) ;
// THEN
assertThat ( result . timeseries ( ) ) . allSatisfy ( ki - > assertThat ( ki . sample ( ) ) . isNull ( ) ) ;
assertThat ( result . attributes ( ) . get ( AttributeScope . SERVER_SCOPE ) )
. allSatisfy ( ki - > assertThat ( ki . sample ( ) ) . isNull ( ) ) ;
}
@Test
public void testFindAvailableKeysByQueryV2_timeseriesOnly ( ) throws Exception {
// GIVEN
var device = createDevice ( "Test device" ) ;
postTelemetry ( device . getId ( ) , new BasicTsKvEntry ( System . currentTimeMillis ( ) , new DoubleDataEntry ( "temperature" , 10 . 0 ) ) ) ;
postAttributes ( device . getId ( ) , AttributeScope . SERVER_SCOPE , new StringDataEntry ( "firmware" , "v1.0" ) ) ;
// WHEN
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2 (
buildDeviceQuery ( "Test device" ) , true , false , null , false ) ;
// THEN
assertThat ( result . timeseries ( ) ) . extracting ( KeyInfo : : key ) . contains ( "temperature" ) ;
assertThat ( result . attributes ( ) ) . isNull ( ) ;
}
@Test
public void testFindAvailableKeysByQueryV2_attributesOnly ( ) throws Exception {
// GIVEN
var device = createDevice ( "Test device" ) ;
postTelemetry ( device . getId ( ) , new BasicTsKvEntry ( System . currentTimeMillis ( ) , new DoubleDataEntry ( "temperature" , 10 . 0 ) ) ) ;
postAttributes ( device . getId ( ) , AttributeScope . SERVER_SCOPE , new StringDataEntry ( "firmware" , "v1.0" ) ) ;
// WHEN
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2 (
buildDeviceQuery ( "Test device" ) , false , true , null , false ) ;
// THEN
assertThat ( result . timeseries ( ) ) . isNull ( ) ;
assertThat ( result . attributes ( ) . get ( AttributeScope . SERVER_SCOPE ) )
. extracting ( KeyInfo : : key ) . contains ( "firmware" ) ;
}
@Test
public void testFindAvailableKeysByQueryV2_noMatchingEntities ( ) throws Exception {
// WHEN
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2 (
buildDeviceQuery ( "NonExistentDevice_" + UUID . randomUUID ( ) ) , true , true , null , true ) ;
// THEN
assertThat ( result . entityTypes ( ) ) . isEmpty ( ) ;
assertThat ( result . timeseries ( ) ) . isEmpty ( ) ;
assertThat ( result . attributes ( ) ) . isEmpty ( ) ;
}
@Test
public void testFindAvailableKeysByQueryV2_assetUsesServerScopeOnly ( ) throws Exception {
// GIVEN
var asset = new Asset ( ) ;
asset . setName ( "Test asset" ) ;
asset . setType ( "default" ) ;
asset = doPost ( "/api/asset" , asset , Asset . class ) ;
postAttributes ( asset . getId ( ) , AttributeScope . SERVER_SCOPE , new StringDataEntry ( "location" , "warehouse" ) ) ;
// WHEN
var filter = new SingleEntityFilter ( ) ;
filter . setSingleEntity ( AliasEntityId . fromEntityId ( asset . getId ( ) ) ) ;
var query = new EntityDataQuery ( filter , new EntityDataPageLink ( 1 , 0 , null , null ) , Collections . emptyList ( ) , null , null ) ;
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2 ( query , false , true , null , false ) ;
// THEN
assertThat ( result . entityTypes ( ) ) . containsExactly ( EntityType . ASSET ) ;
assertThat ( result . attributes ( ) ) . containsOnlyKeys ( AttributeScope . SERVER_SCOPE ) ;
assertThat ( result . attributes ( ) . get ( AttributeScope . SERVER_SCOPE ) )
. extracting ( KeyInfo : : key ) . containsExactly ( "location" ) ;
}
@Test
public void testFindAvailableKeysByQueryV2_rejectsWhenNoKeyTypeRequested ( ) throws Exception {
// WHEN / THEN
EntityDataQuery query = buildDeviceQuery ( "NonExistent" ) ;
doPostAsync ( "/api/v2/entitiesQuery/find/keys?includeTimeseries=false&includeAttributes=false" ,
query , 30_000L ) . andExpect ( status ( ) . isBadRequest ( ) ) ;
}
private AvailableEntityKeysV2 findAvailableEntityKeysByQueryV2 ( EntityDataQuery query ,
boolean includeTimeseries , boolean includeAttributes ,
List < AttributeScope > scopes , boolean includeSamples ) throws Exception {
StringBuilder url = new StringBuilder ( "/api/v2/entitiesQuery/find/keys?" )
. append ( "includeTimeseries=" ) . append ( includeTimeseries )
. append ( "&includeAttributes=" ) . append ( includeAttributes )
. append ( "&includeSamples=" ) . append ( includeSamples ) ;
if ( scopes ! = null ) {
for ( AttributeScope scope : scopes ) {
url . append ( "&scopes=" ) . append ( scope ) ;
}
}
return doPostAsyncWithTypedResponse ( url . toString ( ) , query ,
new TypeReference < > ( ) { } , status ( ) . isOk ( ) ) ;
}
private static void assertKeySample ( List < KeyInfo > keys , String expectedKey , JsonNode expectedValue , long expectedTs ) {
KeyInfo keyInfo = findKeyInfo ( keys , expectedKey ) ;
assertThat ( keyInfo . sample ( ) ) . isNotNull ( ) ;
assertThat ( keyInfo . sample ( ) . value ( ) ) . isEqualTo ( expectedValue ) ;
assertThat ( keyInfo . sample ( ) . ts ( ) ) . isEqualTo ( expectedTs ) ;
}
private static void assertKeySample ( List < KeyInfo > keys , String expectedKey , JsonNode expectedValue ) {
KeyInfo keyInfo = findKeyInfo ( keys , expectedKey ) ;
assertThat ( keyInfo . sample ( ) ) . isNotNull ( ) ;
assertThat ( keyInfo . sample ( ) . value ( ) ) . isEqualTo ( expectedValue ) ;
assertThat ( keyInfo . sample ( ) . ts ( ) ) . isGreaterThan ( 0 ) ;
}
private static KeyInfo findKeyInfo ( List < KeyInfo > keys , String key ) {
return keys . stream ( )
. filter ( ki - > ki . key ( ) . equals ( key ) ) . findFirst ( ) . orElseThrow ( ) ;
}
private static EntityDataQuery buildDeviceQuery ( String deviceName ) {
var filter = new DeviceTypeFilter ( ) ;
filter . setDeviceTypes ( Collections . singletonList ( "default" ) ) ;
filter . setDeviceNameFilter ( deviceName ) ;
return new EntityDataQuery ( filter , new EntityDataPageLink ( 1 , 0 , null , null ) , Collections . emptyList ( ) , null , null ) ;
}
}
Dmytro Skarzhynets
3 months ago