@ -26,7 +26,6 @@ import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardExecutors ;
import org.thingsboard.server.common.data.cf.configuration.Argument ;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType ;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration ;
import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration ;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggSource ;
import org.thingsboard.server.common.data.cf.configuration.aggregation.LatestValuesAggregationCalculatedFieldConfiguration ;
@ -40,6 +39,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery ;
import org.thingsboard.server.common.data.kv.TsKvEntry ;
import org.thingsboard.server.common.data.relation.EntityRelation ;
import org.thingsboard.server.common.data.relation.ProfileEntityRelationPathQuery ;
import org.thingsboard.server.common.data.relation.RelationPathLevel ;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration ;
import org.thingsboard.server.dao.attributes.AttributesService ;
@ -51,10 +51,11 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry ;
import org.thingsboard.server.service.cf.ctx.state.aggregation.AggSingleArgumentEntry ;
import java.util.Collection ;
import java.util.ArrayList ;
import java.util.HashMap ;
import java.util.List ;
import java.util.Map ;
import java.util.Objects ;
import java.util.Optional ;
import java.util.Set ;
import java.util.concurrent.ExecutionException ;
@ -105,7 +106,7 @@ public abstract class AbstractCalculatedFieldProcessingService {
}
yield futures ;
}
case LATEST_VALUES_AGGREGATION - > fetchAggregation ArgumentFuture s ( ctx , entityId ) ;
case LATEST_VALUES_AGGREGATION - > fetchAggArguments ( ctx , entityId , ts ) ;
} ;
return Futures . whenAllComplete ( argFutures . values ( ) )
. call ( ( ) - > resolveArgumentFutures ( argFutures ) ,
@ -122,17 +123,32 @@ public abstract class AbstractCalculatedFieldProcessingService {
return resolveOwnerArgument ( tenantId , entityId ) ;
}
private List < EntityId > resolveRelatedEntities ( TenantId tenantId , EntityId entityId , AggSource aggSource ) {
private ListenableFuture < List < EntityId > > resolveRelatedEntities ( TenantId tenantId , EntityId entityId , AggSource aggSource ) {
RelationPathLevel relation = aggSource . getRelation ( ) ;
return switch ( relation . direction ( ) ) {
case FROM - > aggSource . getEntityProfiles ( ) . stream ( )
. map ( profile - > relationService . findByFromAndTypeAndEntityProfile ( tenantId , entityId , relation . relationType ( ) , profile ) )
. flatMap ( Collection : : stream )
. map ( EntityRelation : : getTo )
List < ListenableFuture < List < EntityRelation > > > relationListsFut = new ArrayList < > ( ) ;
if ( aggSource . getEntityProfiles ( ) . isEmpty ( ) ) {
relationListsFut . add ( relationService . findByProfileEntityRelationPathQueryAsync ( tenantId , new ProfileEntityRelationPathQuery ( entityId , relation , null ) ) ) ;
} else {
aggSource . getEntityProfiles ( ) . forEach ( profile - > relationListsFut . add ( relationService . findByProfileEntityRelationPathQueryAsync ( tenantId , new ProfileEntityRelationPathQuery ( entityId , relation , profile ) ) ) ) ;
}
return Futures . transform ( Futures . allAsList ( relationListsFut ) , relationLists - > {
if ( relationLists = = null ) {
return new ArrayList < > ( ) ;
}
List < EntityRelation > allRelations = relationLists . stream ( )
. filter ( Objects : : nonNull )
. flatMap ( List : : stream )
. toList ( ) ;
case TO - >
aggSource . getEntityProfiles ( ) . stream ( ) . map ( profile - > relationService . findByToAndTypeAndEntityProfile ( tenantId , entityId , relation . relationType ( ) , profile ) . getFrom ( ) ) . toList ( ) ;
} ;
return switch ( relation . direction ( ) ) {
case FROM - > allRelations . stream ( )
. map ( EntityRelation : : getTo )
. toList ( ) ;
case TO - > allRelations . isEmpty ( ) ? List . of ( ) : List . of ( allRelations . get ( 0 ) . getFrom ( ) ) ;
} ;
} , calculatedFieldCallbackExecutor ) ;
}
protected Map < String , ArgumentEntry > resolveArgumentFutures ( Map < String , ListenableFuture < ArgumentEntry > > argFutures ) {
@ -174,6 +190,34 @@ public abstract class AbstractCalculatedFieldProcessingService {
return argFutures ;
}
protected Map < String , ListenableFuture < ArgumentEntry > > fetchAggArguments ( CalculatedFieldCtx ctx , EntityId entityId , long ts ) {
LatestValuesAggregationCalculatedFieldConfiguration aggConfig = ( LatestValuesAggregationCalculatedFieldConfiguration ) ctx . getCalculatedField ( ) . getConfiguration ( ) ;
ListenableFuture < List < EntityId > > relatedEntities = resolveRelatedEntities ( ctx . getTenantId ( ) , entityId , aggConfig . getSource ( ) ) ;
Map < String , ListenableFuture < ArgumentEntry > > futures = new HashMap < > ( ) ;
aggConfig . getInputs ( ) . forEach ( ( key , refKey ) - > {
Argument argument = new Argument ( ) ;
argument . setRefEntityKey ( refKey ) ;
futures . put ( key , Futures . transformAsync ( relatedEntities , entityIds - > fetchAggArgumentEntry ( ctx . getTenantId ( ) , entityIds , argument , System . currentTimeMillis ( ) ) , MoreExecutors . directExecutor ( ) ) ) ;
} ) ;
return futures ;
}
protected ListenableFuture < Map < String , ArgumentEntry > > fetchEntityAggArguments ( CalculatedFieldCtx ctx , EntityId entityId , long ts ) {
LatestValuesAggregationCalculatedFieldConfiguration aggConfig = ( LatestValuesAggregationCalculatedFieldConfiguration ) ctx . getCalculatedField ( ) . getConfiguration ( ) ;
Map < String , ListenableFuture < ArgumentEntry > > futures = new HashMap < > ( ) ;
aggConfig . getInputs ( ) . forEach ( ( key , refKey ) - > {
Argument argument = new Argument ( ) ;
argument . setRefEntityKey ( refKey ) ;
ListenableFuture < ArgumentEntry > argEntryFut = fetchSingleAggArgumentEntry ( ctx . getTenantId ( ) , entityId , argument , ts ) ;
futures . put ( key , argEntryFut ) ;
} ) ;
return Futures . whenAllComplete ( futures . values ( ) )
. call ( ( ) - > resolveArgumentFutures ( futures ) ,
MoreExecutors . directExecutor ( ) ) ;
}
private ListenableFuture < List < EntityId > > resolveGeofencingEntityIds ( TenantId tenantId , EntityId entityId , Map . Entry < String , Argument > entry ) {
Argument value = entry . getValue ( ) ;
if ( value . getRefEntityId ( ) ! = null ) {
@ -197,50 +241,6 @@ public abstract class AbstractCalculatedFieldProcessingService {
return ownerService . getOwner ( tenantId , entityId ) ;
}
private Map < String , ListenableFuture < ArgumentEntry > > fetchAggregationArgumentFutures ( CalculatedFieldCtx ctx , EntityId entityId ) {
LatestValuesAggregationCalculatedFieldConfiguration aggConfig = ( LatestValuesAggregationCalculatedFieldConfiguration ) ctx . getCalculatedField ( ) . getConfiguration ( ) ;
List < EntityId > entityIds = resolveRelatedEntities ( ctx . getTenantId ( ) , entityId , aggConfig . getSource ( ) ) ;
Map < String , ListenableFuture < ArgumentEntry > > futures = new HashMap < > ( ) ;
aggConfig . getInputs ( ) . forEach ( ( key , refKey ) - > {
Argument argument = new Argument ( ) ;
argument . setRefEntityKey ( refKey ) ;
futures . put ( key , fetchAggArgumentEntry ( ctx . getTenantId ( ) , entityIds , argument , System . currentTimeMillis ( ) ) ) ;
} ) ;
return futures ;
}
public ListenableFuture < ArgumentEntry > fetchAggArgumentEntry ( TenantId tenantId , List < EntityId > aggEntities , Argument argument , long startTs ) {
List < ListenableFuture < Map . Entry < EntityId , ? extends ArgumentEntry > > > futures = aggEntities . stream ( )
. map ( entityId - > fetchSingleAggArgumentEntry ( tenantId , entityId , argument , startTs ) )
. toList ( ) ;
ListenableFuture < List < Map . Entry < EntityId , ? extends ArgumentEntry > > > allFutures = Futures . allAsList ( futures ) ;
return Futures . transform ( allFutures ,
entries - > ArgumentEntry . createAggArgument (
entries . stream ( ) . collect ( Collectors . toMap ( Map . Entry : : getKey , Map . Entry : : getValue ) )
) ,
MoreExecutors . directExecutor ( ) ) ;
}
protected ListenableFuture < Map < String , ArgumentEntry > > fetchAggArguments ( CalculatedFieldCtx ctx , EntityId entityId , long ts ) {
CalculatedFieldConfiguration configuration = ctx . getCalculatedField ( ) . getConfiguration ( ) ;
LatestValuesAggregationCalculatedFieldConfiguration aggConfig = ( LatestValuesAggregationCalculatedFieldConfiguration ) configuration ;
Map < String , ListenableFuture < ArgumentEntry > > futures = new HashMap < > ( ) ;
aggConfig . getInputs ( ) . forEach ( ( key , refKey ) - > {
Argument argument = new Argument ( ) ;
argument . setRefEntityKey ( refKey ) ;
ListenableFuture < ArgumentEntry > argumentEntryListenableFuture = fetchAggArgumentEntry ( ctx . getTenantId ( ) , List . of ( entityId ) , argument , System . currentTimeMillis ( ) ) ;
futures . put ( key , argumentEntryListenableFuture ) ;
} ) ;
return Futures . whenAllComplete ( futures . values ( ) )
. call ( ( ) - > resolveArgumentFutures ( futures ) ,
MoreExecutors . directExecutor ( ) ) ;
}
private ListenableFuture < ArgumentEntry > fetchGeofencingKvEntry ( TenantId tenantId , List < EntityId > geofencingEntities , Argument argument ) {
if ( argument . getRefEntityKey ( ) . getType ( ) ! = ArgumentType . ATTRIBUTE ) {
throw new IllegalStateException ( "Unsupported argument key type: " + argument . getRefEntityKey ( ) . getType ( ) ) ;
@ -265,6 +265,22 @@ public abstract class AbstractCalculatedFieldProcessingService {
. collect ( Collectors . toMap ( Map . Entry : : getKey , Map . Entry : : getValue ) ) ) , MoreExecutors . directExecutor ( ) ) ;
}
public ListenableFuture < ArgumentEntry > fetchAggArgumentEntry ( TenantId tenantId , List < EntityId > aggEntities , Argument argument , long startTs ) { List < ListenableFuture < Map . Entry < EntityId , ArgumentEntry > > > futures = aggEntities . stream ( )
. map ( entityId - > {
ListenableFuture < ArgumentEntry > singleAggEntryFut = fetchSingleAggArgumentEntry ( tenantId , entityId , argument , startTs ) ;
return Futures . transform ( singleAggEntryFut , singleAggEntry - > Map . entry ( entityId , singleAggEntry ) , MoreExecutors . directExecutor ( ) ) ;
} )
. toList ( ) ;
ListenableFuture < List < Map . Entry < EntityId , ? extends ArgumentEntry > > > allFutures = Futures . allAsList ( futures ) ;
return Futures . transform ( allFutures ,
entries - > ArgumentEntry . createAggArgument (
entries . stream ( ) . collect ( Collectors . toMap ( Map . Entry : : getKey , Map . Entry : : getValue ) )
) ,
MoreExecutors . directExecutor ( ) ) ;
}
protected ListenableFuture < ArgumentEntry > fetchArgumentValue ( TenantId tenantId , EntityId entityId , Argument argument , long startTs ) {
return switch ( argument . getRefEntityKey ( ) . getType ( ) ) {
case TS_ROLLING - > fetchTsRolling ( tenantId , entityId , argument , startTs ) ;
@ -309,7 +325,15 @@ public abstract class AbstractCalculatedFieldProcessingService {
} , calculatedFieldCallbackExecutor ) ) ;
}
protected ListenableFuture < Map . Entry < EntityId , ? extends ArgumentEntry > > fetchSingleAggArgumentEntry ( TenantId tenantId , EntityId entityId , Argument argument , long startTs ) {
private ReadTsKvQuery buildTsRollingQuery ( TenantId tenantId , Argument argument , long startTs , long endTs ) {
long maxDataPoints = apiLimitService . getLimit (
tenantId , DefaultTenantProfileConfiguration : : getMaxDataPointsPerRollingArg ) ;
int argumentLimit = argument . getLimit ( ) ;
int limit = argumentLimit = = 0 | | argumentLimit > maxDataPoints ? ( int ) maxDataPoints : argumentLimit ;
return new BaseReadTsKvQuery ( argument . getRefEntityKey ( ) . getKey ( ) , startTs , endTs , 0 , limit , Aggregation . NONE ) ;
}
private ListenableFuture < ArgumentEntry > fetchSingleAggArgumentEntry ( TenantId tenantId , EntityId entityId , Argument argument , long startTs ) {
return switch ( argument . getRefEntityKey ( ) . getType ( ) ) {
case TS_ROLLING - > throw new IllegalStateException ( "TS_ROLLING is not supported for aggregation" ) ;
case ATTRIBUTE - > fetchAttributeAggEntry ( tenantId , entityId , argument , startTs ) ;
@ -317,36 +341,26 @@ public abstract class AbstractCalculatedFieldProcessingService {
} ;
}
private ListenableFuture < Map . Entry < EntityId , ? extends ArgumentEntry > > fetchAttributeAggEntry ( TenantId tenantId , EntityId entityId , Argument argument , long defaultLastUpdateTs ) {
private ListenableFuture < ArgumentEntry > fetchAttributeAggEntry ( TenantId tenantId , EntityId entityId , Argument argument , long defaultLastUpdateTs ) {
log . trace ( "[{}][{}] Fetching attribute for key {}" , tenantId , entityId , argument . getRefEntityKey ( ) ) ;
var attributeOptFuture = attributesService . find ( tenantId , entityId , argument . getRefEntityKey ( ) . getScope ( ) , argument . getRefEntityKey ( ) . getKey ( ) ) ;
return Futures . transform ( attributeOptFuture , attrOpt - > {
log . debug ( "[{}][{}] Fetched attribute for key {}: {}" , tenantId , entityId , argument . getRefEntityKey ( ) , attrOpt ) ;
AttributeKvEntry attributeKvEntry = attrOpt . orElseGet ( ( ) - > new BaseAttributeKvEntry ( createDefaultKvEntry ( argument ) , defaultLastUpdateTs , 0L ) ) ;
AggSingleArgumentEntry entry = new AggSingleArgumentEntry ( entityId , attributeKvEntry ) ;
return Map . entry ( entityId , entry ) ;
return new AggSingleArgumentEntry ( entityId , attributeKvEntry ) ;
} , calculatedFieldCallbackExecutor ) ;
}
protected ListenableFuture < Map . Entry < EntityId , ? extends ArgumentEntry > > fetchTsLatestAggEntry ( TenantId tenantId , EntityId entityId , Argument argument , long defaultTs ) {
private ListenableFuture < ArgumentEntry > fetchTsLatestAggEntry ( TenantId tenantId , EntityId entityId , Argument argument , long defaultTs ) {
String key = argument . getRefEntityKey ( ) . getKey ( ) ;
log . trace ( "[{}][{}] Fetching latest timeseries {}" , tenantId , entityId , key ) ;
return Futures . transform (
timeseriesService . findLatest ( tenantId , entityId , key ) ,
result - > {
log . debug ( "[{}][{}] Fetched latest timeseries {}: {}" , tenantId , entityId , key , result ) ;
Optional < TsKvEntry > tsKvEntry = result . or ( ( ) - > Optional . of ( new BasicTsKvEntry ( System . currentTimeMillis ( ) , createDefaultKvEntry ( argument ) , 0L ) ) ) ;
AggSingleArgumentEntry entry = new AggSingleArgumentEntry ( entityId , tsKvEntry . get ( ) ) ;
return Map . entry ( entityId , entry ) ;
Optional < TsKvEntry > tsKvEntry = result . or ( ( ) - > Optional . of ( new BasicTsKvEntry ( defaultTs , createDefaultKvEntry ( argument ) , 0L ) ) ) ;
return new AggSingleArgumentEntry ( entityId , tsKvEntry . get ( ) ) ;
} , calculatedFieldCallbackExecutor ) ;
}
private ReadTsKvQuery buildTsRollingQuery ( TenantId tenantId , Argument argument , long startTs , long endTs ) {
long maxDataPoints = apiLimitService . getLimit (
tenantId , DefaultTenantProfileConfiguration : : getMaxDataPointsPerRollingArg ) ;
int argumentLimit = argument . getLimit ( ) ;
int limit = argumentLimit = = 0 | | argumentLimit > maxDataPoints ? ( int ) maxDataPoints : argumentLimit ;
return new BaseReadTsKvQuery ( argument . getRefEntityKey ( ) . getKey ( ) , startTs , endTs , 0 , limit , Aggregation . NONE ) ;
}
}