ViacheslavKlimov
1 year ago
366 changed files with 18072 additions and 838 deletions
@ -0,0 +1,70 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.DebugModeUtil; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.service.ContextAwareActor; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.TbActorMsg; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
|
|||
@Slf4j |
|||
public abstract class AbstractCalculatedFieldActor extends ContextAwareActor { |
|||
|
|||
protected final TenantId tenantId; |
|||
|
|||
public AbstractCalculatedFieldActor(ActorSystemContext systemContext, TenantId tenantId) { |
|||
super(systemContext); |
|||
this.tenantId = tenantId; |
|||
} |
|||
|
|||
@Override |
|||
protected boolean doProcess(TbActorMsg msg) { |
|||
if (msg instanceof ToCalculatedFieldSystemMsg cfm) { |
|||
Exception cause; |
|||
try { |
|||
return doProcessCfMsg(cfm); |
|||
} catch (CalculatedFieldException cfe) { |
|||
if (DebugModeUtil.isDebugFailuresAvailable(cfe.getCtx().getCalculatedField())) { |
|||
String message; |
|||
if (cfe.getErrorMessage() != null) { |
|||
message = cfe.getErrorMessage(); |
|||
} else if (cfe.getCause() != null) { |
|||
message = cfe.getCause().getMessage(); |
|||
} else { |
|||
message = "N/A"; |
|||
} |
|||
systemContext.persistCalculatedFieldDebugEvent(tenantId, cfe.getCtx().getCfId(), cfe.getEventEntity(), cfe.getArguments(), cfe.getMsgId(), cfe.getMsgType(), null, message); |
|||
} |
|||
cause = cfe.getCause(); |
|||
} catch (Exception e) { |
|||
logProcessingException(e); |
|||
cause = e; |
|||
} |
|||
cfm.getCallback().onFailure(cause); |
|||
return true; |
|||
} else { |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
abstract void logProcessingException(Exception e); |
|||
|
|||
abstract boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException; |
|||
|
|||
} |
|||
@ -0,0 +1,81 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.TbActorException; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
|
|||
@Slf4j |
|||
public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { |
|||
|
|||
private final CalculatedFieldEntityMessageProcessor processor; |
|||
|
|||
CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { |
|||
super(systemContext, tenantId); |
|||
this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); |
|||
} |
|||
|
|||
@Override |
|||
public void init(TbActorCtx ctx) throws TbActorException { |
|||
super.init(ctx); |
|||
log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); |
|||
try { |
|||
processor.init(ctx); |
|||
log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); |
|||
} catch (Exception e) { |
|||
log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); |
|||
throw new TbActorException("Failed to initialize CF entity actor", e); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { |
|||
switch (msg.getMsgType()) { |
|||
case CF_PARTITIONS_CHANGE_MSG: |
|||
processor.process((CalculatedFieldPartitionChangeMsg) msg); |
|||
break; |
|||
case CF_STATE_RESTORE_MSG: |
|||
processor.process((CalculatedFieldStateRestoreMsg) msg); |
|||
break; |
|||
case CF_ENTITY_INIT_CF_MSG: |
|||
processor.process((EntityInitCalculatedFieldMsg) msg); |
|||
break; |
|||
case CF_ENTITY_DELETE_MSG: |
|||
processor.process((CalculatedFieldEntityDeleteMsg) msg); |
|||
break; |
|||
case CF_ENTITY_TELEMETRY_MSG: |
|||
processor.process((EntityCalculatedFieldTelemetryMsg) msg); |
|||
break; |
|||
case CF_LINKED_TELEMETRY_MSG: |
|||
processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); |
|||
break; |
|||
default: |
|||
return false; |
|||
} |
|||
return true; |
|||
} |
|||
|
|||
@Override |
|||
void logProcessingException(Exception e) { |
|||
log.warn("[{}][{}] Processing failure", tenantId, processor.entityId, e); |
|||
} |
|||
} |
|||
@ -0,0 +1,50 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActor; |
|||
import org.thingsboard.server.actors.TbActorId; |
|||
import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; |
|||
import org.thingsboard.server.actors.TbEntityActorId; |
|||
import org.thingsboard.server.actors.device.DeviceActor; |
|||
import org.thingsboard.server.actors.service.ContextBasedCreator; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
|
|||
public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { |
|||
super(context); |
|||
this.tenantId = tenantId; |
|||
this.entityId = entityId; |
|||
} |
|||
|
|||
@Override |
|||
public TbActorId createActorId() { |
|||
return new TbCalculatedFieldEntityActorId(entityId); |
|||
} |
|||
|
|||
@Override |
|||
public TbActor createActor() { |
|||
return new CalculatedFieldEntityActor(context, tenantId, entityId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,44 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
|
|||
@Data |
|||
public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final TbCallback callback; |
|||
|
|||
public CalculatedFieldEntityDeleteMsg(TenantId tenantId, |
|||
EntityId entityId, |
|||
TbCallback callback) { |
|||
this.tenantId = tenantId; |
|||
this.entityId = entityId; |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_ENTITY_DELETE_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,439 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.SneakyThrows; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.DebugModeUtil; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.StringUtils; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.LinkedList; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.stream.Collectors; |
|||
|
|||
|
|||
/** |
|||
* @author Andrew Shvayka |
|||
*/ |
|||
@Slf4j |
|||
public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { |
|||
// (1 for result persistence + 1 for the state persistence )
|
|||
public static final int CALLBACKS_PER_CF = 2; |
|||
|
|||
final TenantId tenantId; |
|||
final EntityId entityId; |
|||
final CalculatedFieldProcessingService cfService; |
|||
final CalculatedFieldStateService cfStateService; |
|||
final int partition; |
|||
|
|||
TbActorCtx ctx; |
|||
Map<CalculatedFieldId, CalculatedFieldState> states = new HashMap<>(); |
|||
|
|||
CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { |
|||
super(systemContext); |
|||
this.tenantId = tenantId; |
|||
this.entityId = entityId; |
|||
this.cfService = systemContext.getCalculatedFieldProcessingService(); |
|||
this.cfStateService = systemContext.getCalculatedFieldStateService(); |
|||
this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); |
|||
} |
|||
|
|||
void init(TbActorCtx ctx) { |
|||
this.ctx = ctx; |
|||
} |
|||
|
|||
public void process(CalculatedFieldPartitionChangeMsg msg) { |
|||
if (!msg.getPartitions()[partition]) { |
|||
log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); |
|||
ctx.stop(ctx.getSelf()); |
|||
} |
|||
} |
|||
|
|||
public void process(CalculatedFieldStateRestoreMsg msg) { |
|||
CalculatedFieldId cfId = msg.getId().cfId(); |
|||
log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); |
|||
if (msg.getState() != null) { |
|||
states.put(cfId, msg.getState()); |
|||
} else { |
|||
states.remove(cfId); |
|||
} |
|||
} |
|||
|
|||
public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); |
|||
var ctx = msg.getCtx(); |
|||
if (msg.isForceReinit()) { |
|||
log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); |
|||
states.remove(ctx.getCfId()); |
|||
} |
|||
try { |
|||
var state = getOrInitState(ctx); |
|||
if (state.isSizeOk()) { |
|||
processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); |
|||
} else { |
|||
throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); |
|||
} |
|||
} catch (Exception e) { |
|||
if (e instanceof CalculatedFieldException cfe) { |
|||
throw cfe; |
|||
} |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
|||
} |
|||
} |
|||
|
|||
public void process(CalculatedFieldEntityDeleteMsg msg) { |
|||
log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); |
|||
if (this.entityId.equals(msg.getEntityId())) { |
|||
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); |
|||
states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); |
|||
ctx.stop(ctx.getSelf()); |
|||
} else { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
var state = states.remove(cfId); |
|||
if (state != null) { |
|||
cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); |
|||
var proto = msg.getProto(); |
|||
var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); |
|||
MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); |
|||
List<CalculatedFieldId> cfIdList = getCalculatedFieldIds(proto); |
|||
Set<CalculatedFieldId> cfIdSet = new HashSet<>(cfIdList); |
|||
for (var ctx : msg.getEntityIdFields()) { |
|||
process(ctx, proto, cfIdSet, cfIdList, callback); |
|||
} |
|||
for (var ctx : msg.getProfileIdFields()) { |
|||
process(ctx, proto, cfIdSet, cfIdList, callback); |
|||
} |
|||
} |
|||
|
|||
public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); |
|||
var proto = msg.getProto(); |
|||
var ctx = msg.getCtx(); |
|||
var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); |
|||
try { |
|||
List<CalculatedFieldId> cfIds = getCalculatedFieldIds(proto); |
|||
if (cfIds.contains(ctx.getCfId())) { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} else { |
|||
if (proto.getTsDataCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else if (proto.getAttrDataCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else if (proto.getRemovedTsKeysCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else if (proto.getRemovedAttrKeysCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
|||
} |
|||
} |
|||
|
|||
private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection<CalculatedFieldId> cfIds, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
try { |
|||
if (cfIds.contains(ctx.getCfId())) { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} else { |
|||
if (proto.getTsDataCount() > 0) { |
|||
processTelemetry(ctx, proto, cfIdList, callback); |
|||
} else if (proto.getAttrDataCount() > 0) { |
|||
processAttributes(ctx, proto, cfIdList, callback); |
|||
} else if (proto.getRemovedTsKeysCount() > 0) { |
|||
processRemovedTelemetry(ctx, proto, cfIdList, callback); |
|||
} else if (proto.getRemovedAttrKeysCount() > 0) { |
|||
processRemovedAttributes(ctx, proto, cfIdList, callback); |
|||
} else { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
if (e instanceof CalculatedFieldException cfe) { |
|||
throw cfe; |
|||
} |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
|||
} |
|||
} |
|||
|
|||
private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithDefaultValue(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback, |
|||
Map<String, ArgumentEntry> newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { |
|||
if (newArgValues.isEmpty()) { |
|||
log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
CalculatedFieldState state = states.get(ctx.getCfId()); |
|||
boolean justRestored = false; |
|||
if (state == null) { |
|||
state = getOrInitState(ctx); |
|||
justRestored = true; |
|||
} |
|||
if (state.isSizeOk()) { |
|||
if (state.updateState(newArgValues) || justRestored) { |
|||
cfIdList = new ArrayList<>(cfIdList); |
|||
cfIdList.add(ctx.getCfId()); |
|||
processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); |
|||
} else { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
} else { |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); |
|||
} |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { |
|||
CalculatedFieldState state = states.get(ctx.getCfId()); |
|||
if (state != null) { |
|||
return state; |
|||
} else { |
|||
ListenableFuture<CalculatedFieldState> stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); |
|||
// Ugly but necessary. We do not expect to often fetch data from DB. Only once per <Entity, CalculatedField> pair lifetime.
|
|||
// This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low.
|
|||
// Alternatively, we can fetch the state outside the actor system and push separate command to create this actor,
|
|||
// but this will significantly complicate the code.
|
|||
state = stateFuture.get(1, TimeUnit.MINUTES); |
|||
state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); |
|||
states.put(ctx.getCfId(), state); |
|||
} |
|||
return state; |
|||
} |
|||
|
|||
private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { |
|||
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); |
|||
boolean stateSizeOk; |
|||
if (ctx.isInitialized() && state.isReady()) { |
|||
try { |
|||
CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); |
|||
state.checkStateSize(ctxId, ctx.getMaxStateSize()); |
|||
stateSizeOk = state.isSizeOk(); |
|||
if (stateSizeOk) { |
|||
cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); |
|||
if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { |
|||
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); |
|||
} |
|||
} else { |
|||
state.checkStateSize(ctxId, ctx.getMaxStateSize()); |
|||
stateSizeOk = state.isSizeOk(); |
|||
if (stateSizeOk) { |
|||
callback.onSuccess(); // State was updated but no calculation performed;
|
|||
} |
|||
} |
|||
if (stateSizeOk) { |
|||
cfStateService.persistState(ctxId, state, callback); |
|||
} else { |
|||
removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); |
|||
} |
|||
} |
|||
|
|||
private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) { |
|||
// We remove the state, but remember that it is over-sized in a local map.
|
|||
cfStateService.removeState(ctxId, new TbCallback() { |
|||
@Override |
|||
public void onSuccess() { |
|||
callback.onFailure(ex); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(ex); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) { |
|||
return mapToArguments(ctx.getMainEntityArguments(), data); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List<TsKvProto> data) { |
|||
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
return mapToArguments(argNames, data); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, List<TsKvProto> data) { |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
|||
for (TsKvProto item : data) { |
|||
ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); |
|||
String argName = argNames.get(key); |
|||
if (argName != null) { |
|||
arguments.put(argName, new SingleValueArgumentEntry(item)); |
|||
} |
|||
key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); |
|||
argName = argNames.get(key); |
|||
if (argName != null) { |
|||
arguments.put(argName, new SingleValueArgumentEntry(item)); |
|||
} |
|||
} |
|||
return arguments; |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
|||
return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
|||
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
return mapToArguments(argNames, scope, attrDataList); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
|||
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
|||
for (AttributeValueProto item : attrDataList) { |
|||
ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); |
|||
String argName = argNames.get(key); |
|||
if (argName != null) { |
|||
arguments.put(argName, new SingleValueArgumentEntry(item)); |
|||
} |
|||
} |
|||
return arguments; |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List<String> removedAttrKeys) { |
|||
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List<String> removedAttrKeys) { |
|||
return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(Map<ReferencedEntityKey, String> argNames, Map<String, Argument> configArguments, AttributeScopeProto scope, List<String> removedAttrKeys) { |
|||
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
|||
for (String removedKey : removedAttrKeys) { |
|||
ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); |
|||
String argName = argNames.get(key); |
|||
if (argName != null) { |
|||
Argument argument = configArguments.get(argName); |
|||
String defaultValue = (argument != null) ? argument.getDefaultValue() : null; |
|||
arguments.put(argName, StringUtils.isNotEmpty(defaultValue) |
|||
? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) |
|||
: new SingleValueArgumentEntry()); |
|||
|
|||
} |
|||
} |
|||
return arguments; |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List<String> removedTelemetryKeys) { |
|||
Map<String, Argument> deletedArguments = ctx.getArguments().entrySet().stream() |
|||
.filter(entry -> removedTelemetryKeys.contains(entry.getKey())) |
|||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); |
|||
|
|||
Map<String, ArgumentEntry> fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); |
|||
|
|||
fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); |
|||
return fetchedArgs; |
|||
} |
|||
|
|||
private static List<CalculatedFieldId> getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { |
|||
List<CalculatedFieldId> cfIds = new LinkedList<>(); |
|||
for (var cfId : proto.getPreviousCalculatedFieldsList()) { |
|||
cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); |
|||
} |
|||
return cfIds; |
|||
} |
|||
|
|||
private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { |
|||
if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { |
|||
return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); |
|||
} |
|||
return null; |
|||
} |
|||
|
|||
private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { |
|||
if (!proto.getTbMsgType().isEmpty()) { |
|||
return TbMsgType.valueOf(proto.getTbMsgType()); |
|||
} |
|||
return null; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Builder; |
|||
import lombok.Getter; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
|
|||
@Getter |
|||
@Builder |
|||
public class CalculatedFieldException extends Exception { |
|||
|
|||
private final CalculatedFieldCtx ctx; |
|||
private final EntityId eventEntity; |
|||
private final UUID msgId; |
|||
private final TbMsgType msgType; |
|||
private Map<String, ArgumentEntry> arguments; |
|||
private String errorMessage; |
|||
private Exception cause; |
|||
|
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
|
|||
@Data |
|||
public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldLinkedTelemetryMsgProto proto; |
|||
private final TbCallback callback; |
|||
|
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_LINKED_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,90 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.TbActorException; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
|
|||
/** |
|||
* Created by ashvayka on 15.03.18. |
|||
*/ |
|||
@Slf4j |
|||
public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { |
|||
|
|||
private final CalculatedFieldManagerMessageProcessor processor; |
|||
|
|||
public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { |
|||
super(systemContext, tenantId); |
|||
this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); |
|||
} |
|||
|
|||
@Override |
|||
public void init(TbActorCtx ctx) throws TbActorException { |
|||
super.init(ctx); |
|||
log.debug("[{}] Starting CF manager actor.", processor.tenantId); |
|||
try { |
|||
processor.init(ctx); |
|||
log.debug("[{}] CF manager actor started.", processor.tenantId); |
|||
} catch (Exception e) { |
|||
log.warn("[{}] Unknown failure", processor.tenantId, e); |
|||
throw new TbActorException("Failed to initialize manager actor", e); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { |
|||
switch (msg.getMsgType()) { |
|||
case CF_PARTITIONS_CHANGE_MSG: |
|||
processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); |
|||
break; |
|||
case CF_INIT_MSG: |
|||
processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); |
|||
break; |
|||
case CF_LINK_INIT_MSG: |
|||
processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); |
|||
break; |
|||
case CF_STATE_RESTORE_MSG: |
|||
processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); |
|||
break; |
|||
case CF_ENTITY_LIFECYCLE_MSG: |
|||
processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); |
|||
break; |
|||
case CF_TELEMETRY_MSG: |
|||
processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); |
|||
break; |
|||
case CF_LINKED_TELEMETRY_MSG: |
|||
processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); |
|||
break; |
|||
default: |
|||
return false; |
|||
} |
|||
return true; |
|||
} |
|||
|
|||
@Override |
|||
void logProcessingException(Exception e) { |
|||
log.warn("[{}] Processing failure", tenantId, e); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,46 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActor; |
|||
import org.thingsboard.server.actors.TbActorId; |
|||
import org.thingsboard.server.actors.TbEntityActorId; |
|||
import org.thingsboard.server.actors.TbStringActorId; |
|||
import org.thingsboard.server.actors.service.ContextBasedCreator; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { |
|||
|
|||
private final TenantId tenantId; |
|||
|
|||
public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { |
|||
super(context); |
|||
this.tenantId = tenantId; |
|||
} |
|||
|
|||
@Override |
|||
public TbActorId createActorId() { |
|||
return new TbStringActorId("CFM|" + tenantId); |
|||
} |
|||
|
|||
@Override |
|||
public TbActor createActor() { |
|||
return new CalculatedFieldManagerActor(context, tenantId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,471 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.DebugModeUtil; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.TbActorRef; |
|||
import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; |
|||
import org.thingsboard.server.actors.service.DefaultActorService; |
|||
import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.id.AssetId; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
|||
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.CopyOnWriteArrayList; |
|||
|
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; |
|||
|
|||
|
|||
/** |
|||
* @author Andrew Shvayka |
|||
*/ |
|||
@Slf4j |
|||
public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { |
|||
|
|||
private final Map<CalculatedFieldId, CalculatedFieldCtx> calculatedFields = new HashMap<>(); |
|||
private final Map<EntityId, List<CalculatedFieldCtx>> entityIdCalculatedFields = new HashMap<>(); |
|||
private final Map<EntityId, List<CalculatedFieldLink>> entityIdCalculatedFieldLinks = new HashMap<>(); |
|||
|
|||
private final CalculatedFieldProcessingService cfExecService; |
|||
private final CalculatedFieldStateService cfStateService; |
|||
private final CalculatedFieldEntityProfileCache cfEntityCache; |
|||
private final CalculatedFieldService cfDaoService; |
|||
private final TbAssetProfileCache assetProfileCache; |
|||
private final TbDeviceProfileCache deviceProfileCache; |
|||
protected final TenantId tenantId; |
|||
|
|||
protected TbActorCtx ctx; |
|||
|
|||
CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { |
|||
super(systemContext); |
|||
this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); |
|||
this.cfExecService = systemContext.getCalculatedFieldProcessingService(); |
|||
this.cfStateService = systemContext.getCalculatedFieldStateService(); |
|||
this.cfDaoService = systemContext.getCalculatedFieldService(); |
|||
this.assetProfileCache = systemContext.getAssetProfileCache(); |
|||
this.deviceProfileCache = systemContext.getDeviceProfileCache(); |
|||
this.tenantId = tenantId; |
|||
} |
|||
|
|||
void init(TbActorCtx ctx) { |
|||
this.ctx = ctx; |
|||
} |
|||
|
|||
public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing CF init message.", msg.getCf().getId()); |
|||
var cf = msg.getCf(); |
|||
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
|||
try { |
|||
cfCtx.init(); |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
|||
} |
|||
calculatedFields.put(cf.getId(), cfCtx); |
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
|
|||
public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { |
|||
log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); |
|||
var link = msg.getLink(); |
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
|
|||
public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { |
|||
var cfId = msg.getId().cfId(); |
|||
var calculatedField = calculatedFields.get(cfId); |
|||
|
|||
if (calculatedField != null) { |
|||
msg.getState().setRequiredArguments(calculatedField.getArgNames()); |
|||
log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); |
|||
getOrCreateActor(msg.getId().entityId()).tell(msg); |
|||
} else { |
|||
cfStateService.removeState(msg.getId(), msg.getCallback()); |
|||
} |
|||
} |
|||
|
|||
public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { |
|||
log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); |
|||
var entityType = msg.getData().getEntityId().getEntityType(); |
|||
var event = msg.getData().getEvent(); |
|||
switch (entityType) { |
|||
case CALCULATED_FIELD: { |
|||
switch (event) { |
|||
case CREATED: |
|||
onCfCreated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case UPDATED: |
|||
onCfUpdated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case DELETED: |
|||
onCfDeleted(msg.getData(), msg.getCallback()); |
|||
break; |
|||
default: |
|||
msg.getCallback().onSuccess(); |
|||
break; |
|||
} |
|||
break; |
|||
} |
|||
case DEVICE: |
|||
case ASSET: { |
|||
switch (event) { |
|||
case CREATED: |
|||
onEntityCreated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case UPDATED: |
|||
onEntityUpdated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case DELETED: |
|||
onEntityDeleted(msg.getData(), msg.getCallback()); |
|||
break; |
|||
default: |
|||
msg.getCallback().onSuccess(); |
|||
break; |
|||
} |
|||
break; |
|||
} |
|||
default: { |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
EntityId entityId = msg.getEntityId(); |
|||
EntityId profileId = getProfileId(tenantId, entityId); |
|||
cfEntityCache.add(tenantId, profileId, entityId); |
|||
var entityIdFields = getCalculatedFieldsByEntityId(entityId); |
|||
var profileIdFields = getCalculatedFieldsByEntityId(profileId); |
|||
var fieldsCount = entityIdFields.size() + profileIdFields.size(); |
|||
if (fieldsCount > 0) { |
|||
MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); |
|||
entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
|||
profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { |
|||
cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); |
|||
var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); |
|||
var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); |
|||
var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); |
|||
if (fieldsCount > 0) { |
|||
MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); |
|||
var entityId = msg.getEntityId(); |
|||
oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), callback)); |
|||
newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
cfEntityCache.evict(tenantId, msg.getEntityId()); |
|||
log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); |
|||
getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); |
|||
} |
|||
|
|||
private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
if (calculatedFields.containsKey(cfId)) { |
|||
log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
var cf = cfDaoService.findById(msg.getTenantId(), cfId); |
|||
if (cf == null) { |
|||
log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
|||
try { |
|||
cfCtx.init(); |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
|||
} |
|||
calculatedFields.put(cf.getId(), cfCtx); |
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); |
|||
addLinks(cf); |
|||
initCf(cfCtx, callback, false); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
var oldCfCtx = calculatedFields.get(cfId); |
|||
if (oldCfCtx == null) { |
|||
onCfCreated(msg, callback); |
|||
} else { |
|||
var newCf = cfDaoService.findById(msg.getTenantId(), cfId); |
|||
if (newCf == null) { |
|||
log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
|||
try { |
|||
newCfCtx.init(); |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
|||
} |
|||
calculatedFields.put(newCf.getId(), newCfCtx); |
|||
List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); |
|||
List<CalculatedFieldCtx> newCfList = new CopyOnWriteArrayList<>(); |
|||
boolean found = false; |
|||
for (CalculatedFieldCtx oldCtx : oldCfList) { |
|||
if (oldCtx.getCfId().equals(newCf.getId())) { |
|||
newCfList.add(newCfCtx); |
|||
found = true; |
|||
} else { |
|||
newCfList.add(oldCtx); |
|||
} |
|||
} |
|||
if (!found) { |
|||
newCfList.add(newCfCtx); |
|||
} |
|||
entityIdCalculatedFields.put(newCf.getEntityId(), newCfList); |
|||
|
|||
deleteLinks(oldCfCtx); |
|||
addLinks(newCf); |
|||
|
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); |
|||
if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { |
|||
initCf(newCfCtx, callback, stateChanges); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
var cfCtx = calculatedFields.remove(cfId); |
|||
if (cfCtx == null) { |
|||
log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); |
|||
deleteLinks(cfCtx); |
|||
|
|||
EntityId entityId = cfCtx.getEntityId(); |
|||
EntityType entityType = cfCtx.getEntityId().getEntityType(); |
|||
if (isProfileEntity(entityType)) { |
|||
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); |
|||
if (!entityIds.isEmpty()) { |
|||
//TODO: no need to do this if we cache all created actors and know which one belong to us;
|
|||
var multiCallback = new MultipleTbCallback(entityIds.size(), callback); |
|||
entityIds.forEach(id -> deleteCfForEntity(entityId, cfId, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} else { |
|||
deleteCfForEntity(entityId, cfId, callback); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { |
|||
EntityId entityId = msg.getEntityId(); |
|||
log.info("Received telemetry msg from entity [{}]", entityId); |
|||
// 2 = 1 for CF processing + 1 for links processing
|
|||
MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); |
|||
// process all cfs related to entity, or it's profile;
|
|||
var entityIdFields = getCalculatedFieldsByEntityId(entityId); |
|||
var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); |
|||
if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { |
|||
log.info("Pushing telemetry msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
// process all links (if any);
|
|||
List<CalculatedFieldEntityCtxId> linkedCalculatedFields = filterCalculatedFieldLinks(msg); |
|||
var linksSize = linkedCalculatedFields.size(); |
|||
if (linksSize > 0) { |
|||
cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { |
|||
EntityId sourceEntityId = msg.getEntityId(); |
|||
log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); |
|||
var proto = msg.getProto(); |
|||
var linksList = proto.getLinksList(); |
|||
for (var linkProto : linksList) { |
|||
var link = fromProto(linkProto); |
|||
var targetEntityId = link.entityId(); |
|||
var targetEntityType = targetEntityId.getEntityType(); |
|||
var cf = calculatedFields.get(link.cfId()); |
|||
if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { |
|||
// iterate over all entities that belong to profile and push the message for corresponding CF
|
|||
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); |
|||
if (!entityIds.isEmpty()) { |
|||
MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); |
|||
var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); |
|||
entityIds.forEach(entityId -> { |
|||
log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(newMsg); |
|||
}); |
|||
} else { |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
} else { |
|||
log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); |
|||
var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); |
|||
getOrCreateActor(targetEntityId).tell(newMsg); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private List<CalculatedFieldEntityCtxId> filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { |
|||
EntityId entityId = msg.getEntityId(); |
|||
var proto = msg.getProto(); |
|||
List<CalculatedFieldEntityCtxId> result = new ArrayList<>(); |
|||
for (var link : getCalculatedFieldLinksByEntityId(entityId)) { |
|||
CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); |
|||
if (ctx.linkMatches(entityId, proto)) { |
|||
result.add(ctx.toCalculatedFieldEntityCtxId()); |
|||
} |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private List<CalculatedFieldCtx> getCalculatedFieldsByEntityId(EntityId entityId) { |
|||
if (entityId == null) { |
|||
return Collections.emptyList(); |
|||
} |
|||
var result = entityIdCalculatedFields.get(entityId); |
|||
if (result == null) { |
|||
result = Collections.emptyList(); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId) { |
|||
if (entityId == null) { |
|||
return Collections.emptyList(); |
|||
} |
|||
var result = entityIdCalculatedFieldLinks.get(entityId); |
|||
if (result == null) { |
|||
result = Collections.emptyList(); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { |
|||
EntityId entityId = cfCtx.getEntityId(); |
|||
EntityType entityType = cfCtx.getEntityId().getEntityType(); |
|||
if (isProfileEntity(entityType)) { |
|||
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); |
|||
if (!entityIds.isEmpty()) { |
|||
var multiCallback = new MultipleTbCallback(entityIds.size(), callback); |
|||
entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} else { |
|||
initCfForEntity(entityId, cfCtx, forceStateReinit, callback); |
|||
} |
|||
} |
|||
|
|||
private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { |
|||
log.info("Pushing delete CF msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); |
|||
} |
|||
|
|||
private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { |
|||
log.info("Pushing entity init CF msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); |
|||
} |
|||
|
|||
private static boolean isProfileEntity(EntityType entityType) { |
|||
return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); |
|||
} |
|||
|
|||
private EntityId getProfileId(TenantId tenantId, EntityId entityId) { |
|||
return switch (entityId.getEntityType()) { |
|||
case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); |
|||
case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); |
|||
default -> null; |
|||
}; |
|||
} |
|||
|
|||
private TbActorRef getOrCreateActor(EntityId entityId) { |
|||
return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), |
|||
() -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, |
|||
() -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), |
|||
() -> true); |
|||
} |
|||
|
|||
private void addLinks(CalculatedField newCf) { |
|||
var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); |
|||
newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link)); |
|||
} |
|||
|
|||
private void deleteLinks(CalculatedFieldCtx cfCtx) { |
|||
var oldCf = cfCtx.getCalculatedField(); |
|||
var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); |
|||
oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).remove(link)); |
|||
} |
|||
|
|||
public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { |
|||
ctx.broadcastToChildren(msg, true); |
|||
} |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
@Data |
|||
public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final CalculatedFieldEntityCtxId id; |
|||
private final CalculatedFieldState state; |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_STATE_RESTORE_MSG; |
|||
} |
|||
|
|||
@Override |
|||
public TenantId getTenantId() { |
|||
return id.tenantId(); |
|||
} |
|||
} |
|||
@ -0,0 +1,39 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
|
|||
@Data |
|||
public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldTelemetryMsgProto proto; |
|||
private final TbCallback callback; |
|||
|
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,42 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Data |
|||
public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldTelemetryMsgProto proto; |
|||
private final CalculatedFieldCtx ctx; |
|||
private final TbCallback callback; |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_LINKED_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Data |
|||
public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldTelemetryMsgProto proto; |
|||
// Both lists are effectively immutable in CalculatedFieldManagerMessageProcessor and must stay so.
|
|||
private final List<CalculatedFieldCtx> entityIdFields; |
|||
private final List<CalculatedFieldCtx> profileIdFields; |
|||
private final TbCallback callback; |
|||
|
|||
public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, |
|||
List<CalculatedFieldCtx> entityIdFields, |
|||
List<CalculatedFieldCtx> profileIdFields, |
|||
TbCallback callback) { |
|||
this.tenantId = msg.getTenantId(); |
|||
this.entityId = msg.getEntityId(); |
|||
this.proto = msg.getProto(); |
|||
this.entityIdFields = entityIdFields; |
|||
this.profileIdFields = profileIdFields; |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_ENTITY_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,41 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Data |
|||
public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final CalculatedFieldCtx ctx; |
|||
private final TbCallback callback; |
|||
private final boolean forceReinit; |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_ENTITY_INIT_CF_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
|
|||
import java.util.UUID; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
@Slf4j |
|||
public class MultipleTbCallback implements TbCallback { |
|||
@Getter |
|||
private final UUID id; |
|||
private final AtomicInteger counter; |
|||
private final TbCallback callback; |
|||
|
|||
public MultipleTbCallback(int count, TbCallback callback) { |
|||
id = UUID.randomUUID(); |
|||
this.counter = new AtomicInteger(count); |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public void onSuccess() { |
|||
onSuccess(1); |
|||
} |
|||
|
|||
public void onSuccess(int number) { |
|||
log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); |
|||
if (counter.addAndGet(-number) <= 0) { |
|||
log.trace("[{}][{}] Done.", id, callback.getId()); |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.warn("[{}][{}] onFailure.", id, callback.getId()); |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
@ -0,0 +1,269 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import com.fasterxml.jackson.core.type.TypeReference; |
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.fasterxml.jackson.databind.node.ObjectNode; |
|||
import io.swagger.v3.oas.annotations.Parameter; |
|||
import io.swagger.v3.oas.annotations.media.Schema; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.http.HttpStatus; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.PathVariable; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RequestMethod; |
|||
import org.springframework.web.bind.annotation.RequestParam; |
|||
import org.springframework.web.bind.annotation.ResponseBody; |
|||
import org.springframework.web.bind.annotation.ResponseStatus; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.EventInfo; |
|||
import org.thingsboard.server.common.data.HasTenantId; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.event.EventType; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.HasId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.config.annotations.ApiOperation; |
|||
import org.thingsboard.server.dao.event.EventService; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; |
|||
import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
import org.thingsboard.server.service.security.permission.Operation; |
|||
import org.thingsboard.server.service.security.permission.Resource; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Objects; |
|||
import java.util.Optional; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; |
|||
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; |
|||
import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; |
|||
import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; |
|||
import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api") |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class CalculatedFieldController extends BaseController { |
|||
|
|||
private final TbCalculatedFieldService tbCalculatedFieldService; |
|||
private final EventService eventService; |
|||
private final TbelInvokeService tbelInvokeService; |
|||
|
|||
public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; |
|||
|
|||
public static final int TIMEOUT = 20; |
|||
|
|||
private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" |
|||
+ MARKDOWN_CODE_BLOCK_START |
|||
+ "{\n" + |
|||
" \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" + |
|||
" \"arguments\": {\n" + |
|||
" \"temperature\": {\n" + |
|||
" \"type\": \"TS_ROLLING\",\n" + |
|||
" \"timeWindow\": {\n" + |
|||
" \"startTs\": 1739775630002,\n" + |
|||
" \"endTs\": 65432211,\n" + |
|||
" \"limit\": 5\n" + |
|||
" },\n" + |
|||
" \"values\": [\n" + |
|||
" { \"ts\": 1739775639851, \"value\": 23 },\n" + |
|||
" { \"ts\": 1739775664561, \"value\": 43 },\n" + |
|||
" { \"ts\": 1739775713079, \"value\": 15 },\n" + |
|||
" { \"ts\": 1739775999522, \"value\": 34 },\n" + |
|||
" { \"ts\": 1739776228452, \"value\": 22 }\n" + |
|||
" ]\n" + |
|||
" },\n" + |
|||
" \"humidity\": { \"type\": \"SINGLE_VALUE\", \"ts\": 1739776478057, \"value\": 23 }\n" + |
|||
" }\n" + |
|||
"}" |
|||
+ MARKDOWN_CODE_BLOCK_END |
|||
+ "\n\n Expected result JSON contains \"output\" and \"error\"."; |
|||
|
|||
@ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", |
|||
notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + |
|||
"The newly created Calculated Field Id will be present in the response. " + |
|||
"Specify existing Calculated Field Id to update the calculated field. " + |
|||
"Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + |
|||
"Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " |
|||
+ TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") |
|||
@RequestBody CalculatedField calculatedField) throws Exception { |
|||
calculatedField.setTenantId(getTenantId()); |
|||
checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); |
|||
checkReferencedEntities(calculatedField.getConfiguration(), getCurrentUser()); |
|||
return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); |
|||
} |
|||
|
|||
@ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", |
|||
notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." |
|||
) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { |
|||
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
|||
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
|||
CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); |
|||
checkNotNull(calculatedField); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); |
|||
return calculatedField; |
|||
} |
|||
|
|||
@ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", |
|||
notes = "Fetch the Calculated Fields based on the provided Entity Id." |
|||
) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public PageData<CalculatedField> getCalculatedFieldsByEntityId( |
|||
@Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, |
|||
@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, |
|||
@Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, |
|||
@Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, |
|||
@Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, |
|||
@Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, |
|||
@Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { |
|||
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); |
|||
checkParameter("entityId", entityIdStr); |
|||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); |
|||
checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); |
|||
return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); |
|||
} |
|||
|
|||
@ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", |
|||
notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) |
|||
@ResponseStatus(value = HttpStatus.OK) |
|||
public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws Exception { |
|||
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
|||
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
|||
CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); |
|||
tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); |
|||
} |
|||
|
|||
@ApiOperation(value = "Get latest calculated field debug event (getLatestCalculatedFieldDebugEvent)", |
|||
notes = "Gets latest calculated field debug event for specified calculated field id. " + |
|||
"Referencing non-existing calculated field id will cause an error. " + TENANT_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/{calculatedFieldId}/debug", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public JsonNode getLatestCalculatedFieldDebugEvent(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { |
|||
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
|||
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
|||
CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.READ); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); |
|||
TenantId tenantId = getCurrentUser().getTenantId(); |
|||
return Optional.ofNullable(eventService.findLatestEvents(tenantId, calculatedFieldId, EventType.DEBUG_CALCULATED_FIELD, 1)) |
|||
.flatMap(events -> events.stream().map(EventInfo::getBody).findFirst()) |
|||
.orElse(null); |
|||
} |
|||
|
|||
@ApiOperation(value = "Test Script expression", |
|||
notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public JsonNode testScript( |
|||
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") |
|||
@RequestBody JsonNode inputParams) { |
|||
String expression = inputParams.get("expression").asText(); |
|||
Map<String, TbelCfArg> arguments = Objects.requireNonNullElse( |
|||
JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference<Map<String, TbelCfArg>>() { |
|||
}), |
|||
Collections.emptyMap() |
|||
); |
|||
ArrayList<String> argNames = new ArrayList<>(arguments.keySet()); |
|||
|
|||
String output = ""; |
|||
String errorText = ""; |
|||
|
|||
try { |
|||
if (tbelInvokeService == null) { |
|||
throw new IllegalArgumentException("TBEL script engine is disabled!"); |
|||
} |
|||
|
|||
CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( |
|||
getTenantId(), |
|||
tbelInvokeService, |
|||
expression, |
|||
argNames.toArray(String[]::new) |
|||
); |
|||
|
|||
Object[] args = argNames.stream() |
|||
.map(arguments::get) |
|||
.toArray(); |
|||
|
|||
JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); |
|||
output = JacksonUtil.toString(json); |
|||
} catch (Exception e) { |
|||
log.error("Error evaluating expression", e); |
|||
errorText = e.getMessage(); |
|||
} |
|||
|
|||
ObjectNode result = JacksonUtil.newObjectNode(); |
|||
result.put("output", output); |
|||
result.put("error", errorText); |
|||
return result; |
|||
} |
|||
|
|||
private <E extends HasId<I> & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { |
|||
List<EntityId> referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); |
|||
for (EntityId referencedEntityId : referencedEntityIds) { |
|||
EntityType entityType = referencedEntityId.getEntityType(); |
|||
switch (entityType) { |
|||
case TENANT, CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); |
|||
default -> throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.exception; |
|||
|
|||
public class CalculatedFieldStateException extends RuntimeException { |
|||
|
|||
public CalculatedFieldStateException(String message) { |
|||
super(message); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,72 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.exception.CalculatedFieldStateException; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; |
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
|||
|
|||
public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { |
|||
|
|||
@Autowired |
|||
private ActorSystemContext actorSystemContext; |
|||
|
|||
protected PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer; |
|||
|
|||
@Override |
|||
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer) { |
|||
this.eventConsumer = eventConsumer; |
|||
} |
|||
|
|||
@Override |
|||
public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { |
|||
if (state.isSizeExceedsLimit()) { |
|||
throw new CalculatedFieldStateException("State size exceeds the maximum allowed limit. The state will not be persisted to RocksDB."); |
|||
} |
|||
doPersist(stateId, toProto(stateId, state), callback); |
|||
} |
|||
|
|||
protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback); |
|||
|
|||
@Override |
|||
public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
|||
doRemove(stateId, callback); |
|||
} |
|||
|
|||
protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); |
|||
|
|||
protected void processRestoredState(CalculatedFieldStateProto stateMsg) { |
|||
var id = fromProto(stateMsg.getId()); |
|||
var state = fromProto(stateMsg); |
|||
processRestoredState(id, state); |
|||
} |
|||
|
|||
protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { |
|||
actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state)); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,45 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
public interface CalculatedFieldCache { |
|||
|
|||
CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); |
|||
|
|||
List<CalculatedField> getCalculatedFieldsByEntityId(EntityId entityId); |
|||
|
|||
List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId); |
|||
|
|||
CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); |
|||
|
|||
List<CalculatedFieldCtx> getCalculatedFieldCtxsByEntityId(EntityId entityId); |
|||
|
|||
void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); |
|||
|
|||
void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); |
|||
|
|||
void evict(CalculatedFieldId calculatedFieldId); |
|||
|
|||
} |
|||
@ -0,0 +1,19 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
public interface CalculatedFieldInitService { |
|||
} |
|||
@ -0,0 +1,43 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
public interface CalculatedFieldProcessingService { |
|||
|
|||
ListenableFuture<CalculatedFieldState> fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); |
|||
|
|||
Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments); |
|||
|
|||
void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List<CalculatedFieldId> cfIds, TbCallback callback); |
|||
|
|||
void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback); |
|||
|
|||
} |
|||
@ -0,0 +1,43 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import org.thingsboard.rule.engine.api.AttributesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.AttributesSaveRequest; |
|||
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; |
|||
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; |
|||
|
|||
import java.util.List; |
|||
|
|||
public interface CalculatedFieldQueueService { |
|||
|
|||
/** |
|||
* Filter CFs based on the request entity. Push to the queue if any matching CF exist; |
|||
* |
|||
* @param request - telemetry save request; |
|||
* @param callback |
|||
*/ |
|||
void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback<Void> callback); |
|||
|
|||
void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback); |
|||
|
|||
void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback); |
|||
|
|||
void pushRequestToQueue(TimeseriesDeleteRequest request, List<String> result, FutureCallback<Void> callback); |
|||
|
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
|
|||
@Data |
|||
public final class CalculatedFieldResult { |
|||
|
|||
private final OutputType type; |
|||
private final AttributeScope scope; |
|||
private final JsonNode result; |
|||
|
|||
} |
|||
@ -0,0 +1,41 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.exception.CalculatedFieldStateException; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
import java.util.Set; |
|||
|
|||
public interface CalculatedFieldStateService { |
|||
|
|||
void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer); |
|||
|
|||
void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; |
|||
|
|||
void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); |
|||
|
|||
void restore(Set<TopicPartitionInfo> partitions); |
|||
|
|||
void stop(); |
|||
|
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import jakarta.annotation.PreDestroy; |
|||
import org.rocksdb.Options; |
|||
import org.rocksdb.WriteOptions; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.server.utils.TbRocksDb; |
|||
|
|||
@Component |
|||
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") |
|||
public class CfRocksDb extends TbRocksDb { |
|||
|
|||
public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { |
|||
super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); |
|||
} |
|||
|
|||
@PreDestroy |
|||
@Override |
|||
public void close() { |
|||
super.close(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,188 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageDataIterable; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.CopyOnWriteArrayList; |
|||
import java.util.concurrent.locks.Lock; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldCache implements CalculatedFieldCache { |
|||
|
|||
private static final Integer UNKNOWN_PARTITION = -1; |
|||
|
|||
private final Lock calculatedFieldFetchLock = new ReentrantLock(); |
|||
|
|||
private final CalculatedFieldService calculatedFieldService; |
|||
private final TbelInvokeService tbelInvokeService; |
|||
private final ActorSystemContext actorSystemContext; |
|||
private final ApiLimitService apiLimitService; |
|||
|
|||
private final ConcurrentMap<CalculatedFieldId, CalculatedField> calculatedFields = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<EntityId, List<CalculatedField>> entityIdCalculatedFields = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<CalculatedFieldId, List<CalculatedFieldLink>> calculatedFieldLinks = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<EntityId, List<CalculatedFieldLink>> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<CalculatedFieldId, CalculatedFieldCtx> calculatedFieldsCtx = new ConcurrentHashMap<>(); |
|||
|
|||
@Value("${calculatedField.initFetchPackSize:50000}") |
|||
@Getter |
|||
private int initFetchPackSize; |
|||
|
|||
@AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) |
|||
public void init() { |
|||
//TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService)
|
|||
PageDataIterable<CalculatedField> cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); |
|||
cfs.forEach(cf -> { |
|||
calculatedFields.putIfAbsent(cf.getId(), cf); |
|||
actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); |
|||
}); |
|||
calculatedFields.values().forEach(cf -> { |
|||
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); |
|||
}); |
|||
PageDataIterable<CalculatedFieldLink> cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); |
|||
cfls.forEach(link -> { |
|||
calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link); |
|||
actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); |
|||
}); |
|||
calculatedFieldLinks.values().stream() |
|||
.flatMap(List::stream) |
|||
.forEach(link -> |
|||
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) |
|||
); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { |
|||
return calculatedFields.get(calculatedFieldId); |
|||
} |
|||
|
|||
@Override |
|||
public List<CalculatedField> getCalculatedFieldsByEntityId(EntityId entityId) { |
|||
return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); |
|||
} |
|||
|
|||
@Override |
|||
public List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId) { |
|||
return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { |
|||
CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); |
|||
if (ctx == null) { |
|||
calculatedFieldFetchLock.lock(); |
|||
try { |
|||
ctx = calculatedFieldsCtx.get(calculatedFieldId); |
|||
if (ctx == null) { |
|||
CalculatedField calculatedField = getCalculatedField(calculatedFieldId); |
|||
if (calculatedField != null) { |
|||
ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); |
|||
ctx.init(); |
|||
calculatedFieldsCtx.put(calculatedFieldId, ctx); |
|||
log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); |
|||
} |
|||
} |
|||
} finally { |
|||
calculatedFieldFetchLock.unlock(); |
|||
} |
|||
} |
|||
log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); |
|||
return ctx; |
|||
} |
|||
|
|||
@Override |
|||
public List<CalculatedFieldCtx> getCalculatedFieldCtxsByEntityId(EntityId entityId) { |
|||
if (entityId == null) { |
|||
return Collections.emptyList(); |
|||
} |
|||
return getCalculatedFieldsByEntityId(entityId).stream() |
|||
.map(cf -> getCalculatedFieldCtx(cf.getId())) |
|||
.toList(); |
|||
} |
|||
|
|||
@Override |
|||
public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { |
|||
calculatedFieldFetchLock.lock(); |
|||
try { |
|||
CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); |
|||
EntityId cfEntityId = calculatedField.getEntityId(); |
|||
|
|||
calculatedFields.put(calculatedFieldId, calculatedField); |
|||
|
|||
entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); |
|||
|
|||
CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); |
|||
calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); |
|||
|
|||
configuration.getReferencedEntities().stream() |
|||
.filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) |
|||
.forEach(referencedEntityId -> { |
|||
entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) |
|||
.add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); |
|||
}); |
|||
} finally { |
|||
calculatedFieldFetchLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { |
|||
evict(calculatedFieldId); |
|||
addCalculatedField(tenantId, calculatedFieldId); |
|||
} |
|||
|
|||
@Override |
|||
public void evict(CalculatedFieldId calculatedFieldId) { |
|||
CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); |
|||
log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); |
|||
calculatedFieldLinks.remove(calculatedFieldId); |
|||
log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); |
|||
entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); |
|||
log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); |
|||
calculatedFieldsCtx.remove(calculatedFieldId); |
|||
log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); |
|||
entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); |
|||
log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,59 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.ProfileEntityIdInfo; |
|||
import org.thingsboard.server.common.data.page.PageDataIterable; |
|||
import org.thingsboard.server.dao.asset.AssetService; |
|||
import org.thingsboard.server.dao.device.DeviceService; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; |
|||
|
|||
@Slf4j |
|||
@Service |
|||
@TbRuleEngineComponent |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { |
|||
|
|||
private final CalculatedFieldEntityProfileCache entityProfileCache; |
|||
private final AssetService assetService; |
|||
private final DeviceService deviceService; |
|||
|
|||
@Value("${calculated_fields.init_fetch_pack_size:50000}") |
|||
@Getter |
|||
private int initFetchPackSize; |
|||
|
|||
@AfterStartUp(order = AfterStartUp.CF_READ_PROFILE_ENTITIES_SERVICE) |
|||
public void initCalculatedFieldDefinitions() { |
|||
PageDataIterable<ProfileEntityIdInfo> deviceIdInfos = new PageDataIterable<>(deviceService::findProfileEntityIdInfos, initFetchPackSize); |
|||
for (ProfileEntityIdInfo idInfo : deviceIdInfos) { |
|||
log.trace("Processing device record: {}", idInfo); |
|||
entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); |
|||
} |
|||
PageDataIterable<ProfileEntityIdInfo> assetIdInfos = new PageDataIterable<>(assetService::findProfileEntityIdInfos, initFetchPackSize); |
|||
for (ProfileEntityIdInfo idInfo : assetIdInfos) { |
|||
log.trace("Processing asset record: {}", idInfo); |
|||
entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,324 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.ListeningExecutorService; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.math.NumberUtils; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.common.util.ThingsBoardExecutors; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
|||
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; |
|||
import org.thingsboard.server.cluster.TbClusterService; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.StringUtils; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.Aggregation; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BooleanDataEntry; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; |
|||
import org.thingsboard.server.common.msg.TbMsg; |
|||
import org.thingsboard.server.common.msg.TbMsgMetaData; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Map.Entry; |
|||
import java.util.Optional; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutionException; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.thingsboard.server.common.data.DataConstants.SCOPE; |
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
|||
|
|||
@TbRuleEngineComponent |
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { |
|||
|
|||
private final AttributesService attributesService; |
|||
private final TimeseriesService timeseriesService; |
|||
private final TbClusterService clusterService; |
|||
private final ApiLimitService apiLimitService; |
|||
private final PartitionService partitionService; |
|||
|
|||
private ListeningExecutorService calculatedFieldCallbackExecutor; |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( |
|||
Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); |
|||
} |
|||
|
|||
@PreDestroy |
|||
public void stop() { |
|||
if (calculatedFieldCallbackExecutor != null) { |
|||
calculatedFieldCallbackExecutor.shutdownNow(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<CalculatedFieldState> fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { |
|||
Map<String, ListenableFuture<ArgumentEntry>> argFutures = new HashMap<>(); |
|||
for (var entry : ctx.getArguments().entrySet()) { |
|||
var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; |
|||
var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); |
|||
argFutures.put(entry.getKey(), argValueFuture); |
|||
} |
|||
return Futures.whenAllComplete(argFutures.values()).call(() -> { |
|||
var result = createStateByType(ctx); |
|||
result.updateState(argFutures.entrySet().stream() |
|||
.collect(Collectors.toMap( |
|||
Entry::getKey, // Keep the key as is
|
|||
entry -> { |
|||
try { |
|||
// Resolve the future to get the value
|
|||
return entry.getValue().get(); |
|||
} catch (ExecutionException | InterruptedException e) { |
|||
throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); |
|||
} |
|||
} |
|||
))); |
|||
return result; |
|||
}, calculatedFieldCallbackExecutor); |
|||
} |
|||
|
|||
@Override |
|||
public Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments) { |
|||
Map<String, ListenableFuture<ArgumentEntry>> argFutures = new HashMap<>(); |
|||
for (var entry : arguments.entrySet()) { |
|||
var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; |
|||
var argValueFuture = fetchKvEntry(tenantId, argEntityId, entry.getValue()); |
|||
argFutures.put(entry.getKey(), argValueFuture); |
|||
} |
|||
return argFutures.entrySet().stream() |
|||
.collect(Collectors.toMap( |
|||
Entry::getKey, // Keep the key as is
|
|||
entry -> { |
|||
try { |
|||
// Resolve the future to get the value
|
|||
return entry.getValue().get(); |
|||
} catch (ExecutionException | InterruptedException e) { |
|||
throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); |
|||
} |
|||
} |
|||
)); |
|||
} |
|||
|
|||
@Override |
|||
public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List<CalculatedFieldId> cfIds, TbCallback callback) { |
|||
try { |
|||
OutputType type = calculatedFieldResult.getType(); |
|||
TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; |
|||
TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; |
|||
TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); |
|||
clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
callback.onSuccess(); |
|||
log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(t); |
|||
} |
|||
}); |
|||
} catch (Exception e) { |
|||
log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); |
|||
callback.onFailure(e); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback) { |
|||
Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts = new HashMap<>(); |
|||
List<CalculatedFieldEntityCtxId> broadcasts = new ArrayList<>(); |
|||
for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { |
|||
var linkEntityId = link.entityId(); |
|||
var linkEntityType = linkEntityId.getEntityType(); |
|||
// Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P.
|
|||
boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); |
|||
if (broadcast) { |
|||
broadcasts.add(link); |
|||
} else { |
|||
TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); |
|||
unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); |
|||
} |
|||
} |
|||
MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); |
|||
if (!broadcasts.isEmpty()) { |
|||
broadcast(broadcasts, msg, linkCallback); |
|||
} else { |
|||
linkCallback.onSuccess(); |
|||
} |
|||
if (!unicasts.isEmpty()) { |
|||
unicast(unicasts, msg, linkCallback); |
|||
} else { |
|||
linkCallback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
private void unicast(Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { |
|||
TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); |
|||
unicasts.forEach((topicPartitionInfo, ctxIds) -> { |
|||
CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); |
|||
clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), |
|||
ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); |
|||
}); |
|||
} |
|||
|
|||
private void broadcast(List<CalculatedFieldEntityCtxId> broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { |
|||
TbQueueCallback callback = new TbCallbackWrapper(mainCallback); |
|||
CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); |
|||
clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); |
|||
} |
|||
|
|||
private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List<CalculatedFieldEntityCtxId> links) { |
|||
Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); |
|||
builder.setMsg(telemetryProto); |
|||
for (CalculatedFieldEntityCtxId link : links) { |
|||
builder.addLinks(toProto(link)); |
|||
} |
|||
return builder.build(); |
|||
} |
|||
|
|||
private ListenableFuture<ArgumentEntry> fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { |
|||
return switch (argument.getRefEntityKey().getType()) { |
|||
case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); |
|||
case ATTRIBUTE -> transformSingleValueArgument( |
|||
Futures.transform( |
|||
attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), |
|||
result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), |
|||
calculatedFieldCallbackExecutor) |
|||
); |
|||
case TS_LATEST -> transformSingleValueArgument( |
|||
Futures.transform( |
|||
timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), |
|||
result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), |
|||
calculatedFieldCallbackExecutor)); |
|||
}; |
|||
} |
|||
|
|||
private ListenableFuture<ArgumentEntry> transformSingleValueArgument(ListenableFuture<Optional<? extends KvEntry>> kvEntryFuture) { |
|||
return Futures.transform(kvEntryFuture, kvEntry -> { |
|||
if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { |
|||
return ArgumentEntry.createSingleValueArgument(kvEntry.get()); |
|||
} else { |
|||
return new SingleValueArgumentEntry(); |
|||
} |
|||
}, calculatedFieldCallbackExecutor); |
|||
} |
|||
|
|||
private ListenableFuture<ArgumentEntry> fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { |
|||
long currentTime = System.currentTimeMillis(); |
|||
long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); |
|||
long startTs = currentTime - timeWindow; |
|||
long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); |
|||
int limit = argument.getLimit() == 0 ? (int) maxDataPoints : argument.getLimit(); |
|||
|
|||
ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); |
|||
ListenableFuture<List<TsKvEntry>> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); |
|||
|
|||
return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? new TsRollingArgumentEntry(limit, timeWindow) : ArgumentEntry.createTsRollingArgument(tsRolling, limit, timeWindow), calculatedFieldCallbackExecutor); |
|||
} |
|||
|
|||
private KvEntry createDefaultKvEntry(Argument argument) { |
|||
String key = argument.getRefEntityKey().getKey(); |
|||
String defaultValue = argument.getDefaultValue(); |
|||
if (StringUtils.isBlank(defaultValue)) { |
|||
return new StringDataEntry(key, null); |
|||
} |
|||
if (NumberUtils.isParsable(defaultValue)) { |
|||
return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); |
|||
} |
|||
if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { |
|||
return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); |
|||
} |
|||
return new StringDataEntry(key, defaultValue); |
|||
} |
|||
|
|||
private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { |
|||
return switch (ctx.getCfType()) { |
|||
case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); |
|||
case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); |
|||
}; |
|||
} |
|||
|
|||
private static class TbCallbackWrapper implements TbQueueCallback { |
|||
private final TbCallback callback; |
|||
|
|||
public TbCallbackWrapper(TbCallback callback) { |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
callback.onSuccess(); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,258 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.rule.engine.api.AttributesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.AttributesSaveRequest; |
|||
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; |
|||
import org.thingsboard.server.cluster.TbClusterService; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.id.AssetId; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
|||
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
|||
|
|||
import java.util.EnumSet; |
|||
import java.util.List; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.function.Predicate; |
|||
import java.util.function.Supplier; |
|||
|
|||
import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; |
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { |
|||
|
|||
public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
} |
|||
}; |
|||
|
|||
private final TbAssetProfileCache assetProfileCache; |
|||
private final TbDeviceProfileCache deviceProfileCache; |
|||
private final CalculatedFieldCache calculatedFieldCache; |
|||
private final TbClusterService clusterService; |
|||
|
|||
private static final Set<EntityType> supportedReferencedEntities = EnumSet.of( |
|||
EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT |
|||
); |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(TimeseriesDeleteRequest request, List<String> result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
|
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, |
|||
Predicate<CalculatedFieldCtx> mainEntityFilter, Predicate<CalculatedFieldCtx> linkedEntityFilter, |
|||
Supplier<ToCalculatedFieldMsg> msg, FutureCallback<Void> callback) { |
|||
boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); |
|||
if (send) { |
|||
clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); |
|||
} else { |
|||
if (callback != null) { |
|||
callback.onSuccess(null); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate<CalculatedFieldCtx> filter, Predicate<CalculatedFieldCtx> linkedEntityFilter) { |
|||
boolean send = false; |
|||
if (supportedReferencedEntities.contains(entityId.getEntityType())) { |
|||
send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); |
|||
if (!send) { |
|||
send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); |
|||
} |
|||
if (!send) { |
|||
send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() |
|||
.map(CalculatedFieldLink::getCalculatedFieldId) |
|||
.map(calculatedFieldCache::getCalculatedFieldCtx) |
|||
.anyMatch(linkedEntityFilter); |
|||
} |
|||
} |
|||
return send; |
|||
} |
|||
|
|||
private EntityId getProfileId(TenantId tenantId, EntityId entityId) { |
|||
return switch (entityId.getEntityType()) { |
|||
case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); |
|||
case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); |
|||
default -> null; |
|||
}; |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { |
|||
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); |
|||
|
|||
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); |
|||
List<TsKvEntry> entries = request.getEntries(); |
|||
List<Long> versions = result.getVersions(); |
|||
for (int i = 0; i < entries.size(); i++) { |
|||
long tsVersion = versions.get(i); |
|||
TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); |
|||
telemetryMsg.addTsData(tsProto); |
|||
} |
|||
msg.setTelemetryMsg(telemetryMsg.build()); |
|||
|
|||
return msg.build(); |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List<Long> versions) { |
|||
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); |
|||
|
|||
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); |
|||
telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); |
|||
List<AttributeKvEntry> entries = request.getEntries(); |
|||
for (int i = 0; i < entries.size(); i++) { |
|||
long attrVersion = versions.get(i); |
|||
AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); |
|||
telemetryMsg.addAttrData(attrProto); |
|||
} |
|||
msg.setTelemetryMsg(telemetryMsg.build()); |
|||
|
|||
return msg.build(); |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesDeleteRequest request, List<String> removedKeys) { |
|||
CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) |
|||
.setScope(AttributeScopeProto.valueOf(request.getScope().name())) |
|||
.addAllRemovedAttrKeys(removedKeys).build(); |
|||
return ToCalculatedFieldMsg.newBuilder() |
|||
.setTelemetryMsg(telemetryMsg) |
|||
.build(); |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesDeleteRequest request, List<String> removedKeys) { |
|||
CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) |
|||
.addAllRemovedTsKeys(removedKeys).build(); |
|||
return ToCalculatedFieldMsg.newBuilder() |
|||
.setTelemetryMsg(telemetryMsg) |
|||
.build(); |
|||
} |
|||
|
|||
private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List<CalculatedFieldId> calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { |
|||
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); |
|||
|
|||
telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); |
|||
telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); |
|||
|
|||
telemetryMsg.setEntityType(entityId.getEntityType().name()); |
|||
telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); |
|||
telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); |
|||
|
|||
if (calculatedFieldIds != null) { |
|||
for (CalculatedFieldId cfId : calculatedFieldIds) { |
|||
telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); |
|||
} |
|||
} |
|||
|
|||
if (tbMsgId != null) { |
|||
telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); |
|||
telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); |
|||
} |
|||
|
|||
if (tbMsgType != null) { |
|||
telemetryMsg.setTbMsgType(tbMsgType.name()); |
|||
} |
|||
|
|||
return telemetryMsg; |
|||
} |
|||
|
|||
private static TbQueueCallback wrap(FutureCallback<Void> callback) { |
|||
if (callback != null) { |
|||
return new FutureCallbackWrapper(callback); |
|||
} else { |
|||
return DUMMY_TB_QUEUE_CALLBACK; |
|||
} |
|||
} |
|||
|
|||
private static class FutureCallbackWrapper implements TbQueueCallback { |
|||
private final FutureCallback<Void> callback; |
|||
|
|||
public FutureCallbackWrapper(FutureCallback<Void> callback) { |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
callback.onSuccess(null); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.cache; |
|||
|
|||
import org.springframework.context.ApplicationListener; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
|
|||
import java.util.Collection; |
|||
|
|||
public interface CalculatedFieldEntityProfileCache extends ApplicationListener<PartitionChangeEvent> { |
|||
|
|||
void add(TenantId tenantId, EntityId profileId, EntityId entityId); |
|||
|
|||
void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); |
|||
|
|||
void evict(TenantId tenantId, EntityId entityId); |
|||
|
|||
Collection<EntityId> getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); |
|||
|
|||
int getEntityIdPartition(TenantId tenantId, EntityId entityId); |
|||
} |
|||
@ -0,0 +1,93 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.cache; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.discovery.TbApplicationEventListener; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
|
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@TbRuleEngineComponent |
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
//TODO ashvayka: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor;
|
|||
public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener<PartitionChangeEvent> implements CalculatedFieldEntityProfileCache { |
|||
|
|||
private static final Integer UNKNOWN = 0; |
|||
private final ConcurrentMap<TenantId, TenantEntityProfileCache> tenantCache = new ConcurrentHashMap<>(); |
|||
private final PartitionService partitionService; |
|||
private volatile List<Integer> myPartitions = Collections.emptyList(); |
|||
|
|||
@Override |
|||
protected void onTbApplicationEvent(PartitionChangeEvent event) { |
|||
myPartitions = event.getCfPartitions().stream() |
|||
.filter(TopicPartitionInfo::isMyPartition) |
|||
.map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); |
|||
//Naive approach that need to be improved.
|
|||
tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); |
|||
} |
|||
|
|||
@Override |
|||
public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { |
|||
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
|||
var partition = tpi.getPartition().orElse(UNKNOWN); |
|||
tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) |
|||
.add(profileId, entityId, partition, tpi.isMyPartition()); |
|||
} |
|||
|
|||
@Override |
|||
public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { |
|||
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
|||
var partition = tpi.getPartition().orElse(UNKNOWN); |
|||
var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); |
|||
//TODO: make this method atomic;
|
|||
cache.remove(oldProfileId, entityId); |
|||
cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); |
|||
} |
|||
|
|||
@Override |
|||
public void evict(TenantId tenantId, EntityId entityId) { |
|||
var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); |
|||
cache.removeEntityId(entityId); |
|||
} |
|||
|
|||
@Override |
|||
public Collection<EntityId> getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { |
|||
return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); |
|||
} |
|||
|
|||
@Override |
|||
public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { |
|||
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
|||
return tpi.getPartition().orElse(UNKNOWN); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,122 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.cache; |
|||
|
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.concurrent.locks.ReadWriteLock; |
|||
import java.util.concurrent.locks.ReentrantReadWriteLock; |
|||
|
|||
public class TenantEntityProfileCache { |
|||
|
|||
private final ReadWriteLock lock = new ReentrantReadWriteLock(); |
|||
private final Map<Integer, Map<EntityId, Set<EntityId>>> allEntities = new HashMap<>(); |
|||
private final Map<EntityId, Set<EntityId>> myEntities = new HashMap<>(); |
|||
|
|||
public void setMyPartitions(List<Integer> myPartitions) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
myEntities.clear(); |
|||
myPartitions.forEach(partitionId -> { |
|||
var map = allEntities.get(partitionId); |
|||
if (map != null) { |
|||
map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); |
|||
} |
|||
}); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void removeProfileId(EntityId profileId) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
// Remove from allEntities
|
|||
allEntities.values().forEach(map -> map.remove(profileId)); |
|||
// Remove from myEntities
|
|||
myEntities.remove(profileId); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void removeEntityId(EntityId entityId) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
// Remove from allEntities
|
|||
allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); |
|||
// Remove from myEntities
|
|||
myEntities.values().forEach(set -> set.remove(entityId)); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void remove(EntityId profileId, EntityId entityId) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
// Remove from allEntities
|
|||
allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); |
|||
// Remove from myEntities
|
|||
removeSafely(myEntities, profileId, entityId); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
if(EntityType.DEVICE.equals(profileId.getEntityType())){ |
|||
throw new RuntimeException("WTF?"); |
|||
} |
|||
if (mine) { |
|||
myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); |
|||
} |
|||
allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public Collection<EntityId> getMyEntityIdsByProfileId(EntityId profileId) { |
|||
lock.readLock().lock(); |
|||
try { |
|||
var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); |
|||
List<EntityId> result = new ArrayList<>(entities.size()); |
|||
result.addAll(entities); |
|||
return result; |
|||
} finally { |
|||
lock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
private void removeSafely(Map<EntityId, Set<EntityId>> map, EntityId profileId, EntityId entityId) { |
|||
var set = map.get(profileId); |
|||
if (set != null) { |
|||
set.remove(entityId); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx; |
|||
|
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
public class CalculatedFieldEntityCtx { |
|||
|
|||
private CalculatedFieldEntityCtxId id; |
|||
private CalculatedFieldState state; |
|||
|
|||
public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { |
|||
this.id = id; |
|||
this.state = state; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,28 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx; |
|||
|
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { |
|||
|
|||
public String toKey() { |
|||
return cfId + "_" + entityId; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,61 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import com.fasterxml.jackson.annotation.JsonSubTypes; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
|
|||
import java.util.List; |
|||
|
|||
@JsonTypeInfo( |
|||
use = JsonTypeInfo.Id.NAME, |
|||
include = JsonTypeInfo.As.PROPERTY, |
|||
property = "type" |
|||
) |
|||
@JsonSubTypes({ |
|||
@JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), |
|||
@JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") |
|||
}) |
|||
public interface ArgumentEntry { |
|||
|
|||
@JsonIgnore |
|||
ArgumentEntryType getType(); |
|||
|
|||
Object getValue(); |
|||
|
|||
boolean updateEntry(ArgumentEntry entry); |
|||
|
|||
boolean isEmpty(); |
|||
|
|||
TbelCfArg toTbelCfArg(); |
|||
|
|||
boolean isForceResetPrevious(); |
|||
|
|||
void setForceResetPrevious(boolean forceResetPrevious); |
|||
|
|||
static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { |
|||
return new SingleValueArgumentEntry(kvEntry); |
|||
} |
|||
|
|||
static ArgumentEntry createTsRollingArgument(List<TsKvEntry> kvEntries, int limit, long timeWindow) { |
|||
return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,20 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
public enum ArgumentEntryType { |
|||
SINGLE_VALUE, TS_ROLLING |
|||
} |
|||
@ -0,0 +1,86 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.utils.CalculatedFieldUtils; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
@Data |
|||
@AllArgsConstructor |
|||
public abstract class BaseCalculatedFieldState implements CalculatedFieldState { |
|||
|
|||
protected List<String> requiredArguments; |
|||
protected Map<String, ArgumentEntry> arguments; |
|||
protected boolean sizeExceedsLimit; |
|||
|
|||
public BaseCalculatedFieldState(List<String> requiredArguments) { |
|||
this.requiredArguments = requiredArguments; |
|||
this.arguments = new HashMap<>(); |
|||
} |
|||
|
|||
public BaseCalculatedFieldState() { |
|||
this(new ArrayList<>(), new HashMap<>(), false); |
|||
} |
|||
|
|||
@Override |
|||
public boolean updateState(Map<String, ArgumentEntry> argumentValues) { |
|||
if (arguments == null) { |
|||
arguments = new HashMap<>(); |
|||
} |
|||
|
|||
boolean stateUpdated = false; |
|||
|
|||
for (Map.Entry<String, ArgumentEntry> entry : argumentValues.entrySet()) { |
|||
String key = entry.getKey(); |
|||
ArgumentEntry newEntry = entry.getValue(); |
|||
ArgumentEntry existingEntry = arguments.get(key); |
|||
|
|||
if (existingEntry == null || newEntry.isForceResetPrevious()) { |
|||
validateNewEntry(newEntry); |
|||
arguments.put(key, newEntry); |
|||
stateUpdated = true; |
|||
} else { |
|||
stateUpdated = existingEntry.updateEntry(newEntry); |
|||
} |
|||
} |
|||
|
|||
return stateUpdated; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isReady() { |
|||
return arguments.keySet().containsAll(requiredArguments) && |
|||
arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); |
|||
} |
|||
|
|||
@Override |
|||
public void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize) { |
|||
if (!sizeExceedsLimit && maxStateSize > 0 && CalculatedFieldUtils.toProto(ctxId, this).getSerializedSize() > maxStateSize) { |
|||
arguments.clear(); |
|||
sizeExceedsLimit = true; |
|||
} |
|||
} |
|||
|
|||
protected abstract void validateNewEntry(ArgumentEntry newEntry); |
|||
|
|||
} |
|||
@ -0,0 +1,277 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import lombok.Data; |
|||
import net.objecthunter.exp4j.Expression; |
|||
import net.objecthunter.exp4j.ExpressionBuilder; |
|||
import org.mvel2.MVEL; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; |
|||
import org.thingsboard.server.common.data.util.TbPair; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Data |
|||
public class CalculatedFieldCtx { |
|||
|
|||
private CalculatedField calculatedField; |
|||
|
|||
private CalculatedFieldId cfId; |
|||
private TenantId tenantId; |
|||
private EntityId entityId; |
|||
private CalculatedFieldType cfType; |
|||
private final Map<String, Argument> arguments; |
|||
private final Map<ReferencedEntityKey, String> mainEntityArguments; |
|||
private final Map<EntityId, Map<ReferencedEntityKey, String>> linkedEntityArguments; |
|||
|
|||
private final Map<TbPair<EntityId, ReferencedEntityKey>, String> referencedEntityKeys; |
|||
private final List<String> argNames; |
|||
private Output output; |
|||
private String expression; |
|||
private TbelInvokeService tbelInvokeService; |
|||
private CalculatedFieldScriptEngine calculatedFieldScriptEngine; |
|||
private ThreadLocal<Expression> customExpression; |
|||
|
|||
private boolean initialized; |
|||
|
|||
private long maxDataPointsPerRollingArg; |
|||
private long maxStateSize; |
|||
|
|||
public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { |
|||
this.calculatedField = calculatedField; |
|||
|
|||
this.cfId = calculatedField.getId(); |
|||
this.tenantId = calculatedField.getTenantId(); |
|||
this.entityId = calculatedField.getEntityId(); |
|||
this.cfType = calculatedField.getType(); |
|||
CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); |
|||
this.arguments = configuration.getArguments(); |
|||
this.mainEntityArguments = new HashMap<>(); |
|||
this.linkedEntityArguments = new HashMap<>(); |
|||
for (Map.Entry<String, Argument> entry : arguments.entrySet()) { |
|||
var refId = entry.getValue().getRefEntityId(); |
|||
var refKey = entry.getValue().getRefEntityKey(); |
|||
if (refId == null) { |
|||
mainEntityArguments.put(refKey, entry.getKey()); |
|||
} else { |
|||
linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); |
|||
} |
|||
} |
|||
this.referencedEntityKeys = arguments.entrySet().stream() |
|||
.collect(Collectors.toMap( |
|||
entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), |
|||
Map.Entry::getKey |
|||
)); |
|||
this.argNames = new ArrayList<>(arguments.keySet()); |
|||
this.output = configuration.getOutput(); |
|||
this.expression = configuration.getExpression(); |
|||
this.tbelInvokeService = tbelInvokeService; |
|||
|
|||
this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); |
|||
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; |
|||
} |
|||
|
|||
public void init() { |
|||
if (CalculatedFieldType.SCRIPT.equals(cfType)) { |
|||
try { |
|||
this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); |
|||
initialized = true; |
|||
} catch (Exception e) { |
|||
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); |
|||
} |
|||
} else { |
|||
if (isValidExpression(expression)) { |
|||
this.customExpression = ThreadLocal.withInitial(() -> |
|||
new ExpressionBuilder(expression) |
|||
.implicitMultiplication(true) |
|||
.variables(this.arguments.keySet()) |
|||
.build() |
|||
); |
|||
initialized = true; |
|||
} else { |
|||
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { |
|||
if (tbelInvokeService == null) { |
|||
throw new IllegalArgumentException("TBEL script engine is disabled!"); |
|||
} |
|||
|
|||
return new CalculatedFieldTbelScriptEngine( |
|||
tenantId, |
|||
tbelInvokeService, |
|||
expression, |
|||
argNames.toArray(String[]::new) |
|||
); |
|||
} |
|||
|
|||
private boolean isValidExpression(String expression) { |
|||
try { |
|||
MVEL.compileExpression(expression); |
|||
return true; |
|||
} catch (Exception e) { |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
public boolean matches(List<AttributeKvEntry> values, AttributeScope scope) { |
|||
return matchesAttributes(mainEntityArguments, values, scope); |
|||
} |
|||
|
|||
public boolean linkMatches(EntityId entityId, List<AttributeKvEntry> values, AttributeScope scope) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesAttributes(map, values, scope); |
|||
} |
|||
|
|||
public boolean matches(List<TsKvEntry> values) { |
|||
return matchesTimeSeries(mainEntityArguments, values); |
|||
} |
|||
|
|||
public boolean linkMatches(EntityId entityId, List<TsKvEntry> values) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesTimeSeries(map, values); |
|||
} |
|||
|
|||
private boolean matchesAttributes(Map<ReferencedEntityKey, String> argMap, List<AttributeKvEntry> values, AttributeScope scope) { |
|||
for (AttributeKvEntry attrKv : values) { |
|||
ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); |
|||
if (argMap.containsKey(attrKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
private boolean matchesTimeSeries(Map<ReferencedEntityKey, String> argMap, List<TsKvEntry> values) { |
|||
for (TsKvEntry tsKv : values) { |
|||
ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); |
|||
if (argMap.containsKey(latestKey)) { |
|||
return true; |
|||
} |
|||
ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); |
|||
if (argMap.containsKey(rollingKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
public boolean matchesKeys(List<String> keys, AttributeScope scope) { |
|||
return matchesAttributesKeys(mainEntityArguments, keys, scope); |
|||
} |
|||
|
|||
public boolean matchesKeys(List<String> keys) { |
|||
return matchesTimeSeriesKeys(mainEntityArguments, keys); |
|||
} |
|||
|
|||
private boolean matchesAttributesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys, AttributeScope scope) { |
|||
for (String key : keys) { |
|||
ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); |
|||
if (argMap.containsKey(attrKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
private boolean matchesTimeSeriesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys) { |
|||
for (String key : keys) { |
|||
ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); |
|||
if (argMap.containsKey(latestKey)) { |
|||
return true; |
|||
} |
|||
ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); |
|||
if (argMap.containsKey(rollingKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
public boolean linkMatchesAttrKeys(EntityId entityId, List<String> keys, AttributeScope scope) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesAttributesKeys(map, keys, scope); |
|||
} |
|||
|
|||
public boolean linkMatchesTsKeys(EntityId entityId, List<String> keys) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesTimeSeriesKeys(map, keys); |
|||
} |
|||
|
|||
public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { |
|||
if (!proto.getTsDataList().isEmpty()) { |
|||
List<TsKvEntry> updatedTelemetry = proto.getTsDataList().stream() |
|||
.map(ProtoUtils::fromProto) |
|||
.toList(); |
|||
return linkMatches(entityId, updatedTelemetry); |
|||
} else if (!proto.getAttrDataList().isEmpty()) { |
|||
AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); |
|||
List<AttributeKvEntry> updatedTelemetry = proto.getAttrDataList().stream() |
|||
.map(ProtoUtils::fromProto) |
|||
.toList(); |
|||
return linkMatches(entityId, updatedTelemetry, scope); |
|||
} else if (!proto.getRemovedTsKeysList().isEmpty()) { |
|||
return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); |
|||
} else { |
|||
return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); |
|||
} |
|||
} |
|||
|
|||
public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { |
|||
return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); |
|||
} |
|||
|
|||
public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { |
|||
boolean expressionChanged = !expression.equals(other.expression); |
|||
boolean outputChanged = !output.equals(other.output); |
|||
return expressionChanged || outputChanged; |
|||
} |
|||
|
|||
public boolean hasStateChanges(CalculatedFieldCtx other) { |
|||
boolean typeChanged = !cfType.equals(other.cfType); |
|||
boolean argumentsChanged = !arguments.equals(other.arguments); |
|||
return typeChanged || argumentsChanged; |
|||
} |
|||
|
|||
public String getSizeExceedsLimitMessage() { |
|||
return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
|
|||
public interface CalculatedFieldScriptEngine { |
|||
|
|||
ListenableFuture<Object> executeScriptAsync(Object[] args); |
|||
|
|||
ListenableFuture<JsonNode> executeJsonAsync(Object[] args); |
|||
|
|||
void destroy(); |
|||
|
|||
} |
|||
@ -0,0 +1,63 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import com.fasterxml.jackson.annotation.JsonSubTypes; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
@JsonTypeInfo( |
|||
use = JsonTypeInfo.Id.NAME, |
|||
include = JsonTypeInfo.As.PROPERTY, |
|||
property = "type" |
|||
) |
|||
@JsonSubTypes({ |
|||
@JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), |
|||
@JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), |
|||
}) |
|||
public interface CalculatedFieldState { |
|||
|
|||
@JsonIgnore |
|||
CalculatedFieldType getType(); |
|||
|
|||
Map<String, ArgumentEntry> getArguments(); |
|||
|
|||
void setRequiredArguments(List<String> requiredArguments); |
|||
|
|||
boolean updateState(Map<String, ArgumentEntry> argumentValues); |
|||
|
|||
ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx); |
|||
|
|||
@JsonIgnore |
|||
boolean isReady(); |
|||
|
|||
boolean isSizeExceedsLimit(); |
|||
|
|||
@JsonIgnore |
|||
default boolean isSizeOk() { |
|||
return !isSizeExceedsLimit(); |
|||
} |
|||
|
|||
void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); |
|||
|
|||
} |
|||
@ -0,0 +1,82 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.script.api.ScriptType; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
import javax.script.ScriptException; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutionException; |
|||
|
|||
@Slf4j |
|||
public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { |
|||
|
|||
private final TbelInvokeService tbelInvokeService; |
|||
|
|||
private final UUID scriptId; |
|||
private final TenantId tenantId; |
|||
|
|||
public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { |
|||
this.tenantId = tenantId; |
|||
this.tbelInvokeService = tbelInvokeService; |
|||
try { |
|||
this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); |
|||
} catch (Exception e) { |
|||
Throwable t = e; |
|||
if (e instanceof ExecutionException) { |
|||
t = e.getCause(); |
|||
} |
|||
throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Object> executeScriptAsync(Object[] args) { |
|||
log.trace("Executing script async, args {}", args); |
|||
return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), |
|||
o -> { |
|||
try { |
|||
return Futures.immediateFuture(o); |
|||
} catch (Exception e) { |
|||
if (e.getCause() instanceof ScriptException) { |
|||
return Futures.immediateFailedFuture(e.getCause()); |
|||
} else if (e.getCause() instanceof RuntimeException) { |
|||
return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); |
|||
} else { |
|||
return Futures.immediateFailedFuture(new ScriptException(e)); |
|||
} |
|||
} |
|||
}, MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<JsonNode> executeJsonAsync(Object[] args) { |
|||
return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
@Override |
|||
public void destroy() { |
|||
tbelInvokeService.release(this.scriptId); |
|||
} |
|||
} |
|||
@ -0,0 +1,157 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueMsgHeaders; |
|||
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.queue.common.consumer.QueueStateService; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; |
|||
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; |
|||
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.Set; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.*; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine') && '${queue.type:null}'=='kafka'") |
|||
public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { |
|||
|
|||
private final TbRuleEngineQueueFactory queueFactory; |
|||
private final PartitionService partitionService; |
|||
|
|||
@Value("${queue.calculated_fields.poll_interval:25}") |
|||
private long pollInterval; |
|||
|
|||
private PartitionedQueueConsumerManager<TbProtoQueueMsg<CalculatedFieldStateProto>> stateConsumer; |
|||
private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>> stateProducer; |
|||
private QueueStateService<TbProtoQueueMsg<ToCalculatedFieldMsg>, TbProtoQueueMsg<CalculatedFieldStateProto>> queueStateService; |
|||
|
|||
private final AtomicInteger counter = new AtomicInteger(); |
|||
|
|||
@Override |
|||
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer) { |
|||
super.init(eventConsumer); |
|||
this.stateConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<CalculatedFieldStateProto>>create() |
|||
.queueKey(QueueKey.CF_STATES) |
|||
.topic(partitionService.getTopic(QueueKey.CF_STATES)) |
|||
.pollInterval(pollInterval) |
|||
.msgPackProcessor((msgs, consumer, config) -> { |
|||
for (TbProtoQueueMsg<CalculatedFieldStateProto> msg : msgs) { |
|||
try { |
|||
if (msg.getValue() != null) { |
|||
processRestoredState(msg.getValue()); |
|||
} else { |
|||
processRestoredState(getStateId(msg.getHeaders()), null); |
|||
} |
|||
} catch (Throwable t) { |
|||
log.error("Failed to process state message: {}", msg, t); |
|||
} |
|||
|
|||
int processedMsgCount = counter.incrementAndGet(); |
|||
if (processedMsgCount % 10000 == 0) { |
|||
log.info("Processed {} calculated field state msgs", processedMsgCount); |
|||
} |
|||
} |
|||
}) |
|||
.consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) |
|||
.consumerExecutor(eventConsumer.getConsumerExecutor()) |
|||
.scheduler(eventConsumer.getScheduler()) |
|||
.taskExecutor(eventConsumer.getTaskExecutor()) |
|||
.build(); |
|||
this.stateProducer = (TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>>) queueFactory.createCalculatedFieldStateProducer(); |
|||
this.queueStateService = new QueueStateService<>(); |
|||
this.queueStateService.init(stateConsumer, super.eventConsumer); |
|||
} |
|||
|
|||
@Override |
|||
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { |
|||
TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); |
|||
TbProtoQueueMsg<CalculatedFieldStateProto> msg = new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto); |
|||
if (stateMsgProto == null) { |
|||
putStateId(msg.getHeaders(), stateId); |
|||
} |
|||
stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
if (callback != null) { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
if (callback != null) { |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
}); |
|||
} |
|||
|
|||
@Override |
|||
protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
|||
doPersist(stateId, null, callback); |
|||
} |
|||
|
|||
@Override |
|||
public void restore(Set<TopicPartitionInfo> partitions) { |
|||
queueStateService.update(partitions); |
|||
} |
|||
|
|||
private void putStateId(TbQueueMsgHeaders headers, CalculatedFieldEntityCtxId stateId) { |
|||
headers.put("tenantId", uuidToBytes(stateId.tenantId().getId())); |
|||
headers.put("cfId", uuidToBytes(stateId.cfId().getId())); |
|||
headers.put("entityId", uuidToBytes(stateId.entityId().getId())); |
|||
headers.put("entityType", stringToBytes(stateId.entityId().getEntityType().name())); |
|||
} |
|||
|
|||
private CalculatedFieldEntityCtxId getStateId(TbQueueMsgHeaders headers) { |
|||
TenantId tenantId = TenantId.fromUUID(bytesToUuid(headers.get("tenantId"))); |
|||
CalculatedFieldId cfId = new CalculatedFieldId(bytesToUuid(headers.get("cfId"))); |
|||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(bytesToString(headers.get("entityType")), bytesToUuid(headers.get("entityId"))); |
|||
return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); |
|||
} |
|||
|
|||
@Override |
|||
public void stop() { |
|||
stateConsumer.stop(); |
|||
stateConsumer.awaitStop(); |
|||
stateProducer.stop(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,73 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.google.protobuf.InvalidProtocolBufferException; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
|||
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.CfRocksDb; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.Set; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") |
|||
public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { |
|||
|
|||
private final CfRocksDb cfRocksDb; |
|||
|
|||
private boolean initialized; |
|||
|
|||
@Override |
|||
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { |
|||
cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); |
|||
callback.onSuccess(); |
|||
} |
|||
|
|||
@Override |
|||
protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
|||
cfRocksDb.delete(stateId.toKey()); |
|||
callback.onSuccess(); |
|||
} |
|||
|
|||
@Override |
|||
public void restore(Set<TopicPartitionInfo> partitions) { |
|||
if (!this.initialized) { |
|||
cfRocksDb.forEach((key, value) -> { |
|||
try { |
|||
processRestoredState(CalculatedFieldStateProto.parseFrom(value)); |
|||
} catch (InvalidProtocolBufferException e) { |
|||
log.error("[{}] Failed to process restored state", key, e); |
|||
} |
|||
}); |
|||
this.initialized = true; |
|||
} |
|||
eventConsumer.update(partitions); |
|||
} |
|||
|
|||
@Override |
|||
public void stop() { |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,68 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Data |
|||
@Slf4j |
|||
@NoArgsConstructor |
|||
public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { |
|||
|
|||
public ScriptCalculatedFieldState(List<String> requiredArguments) { |
|||
super(requiredArguments); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedFieldType getType() { |
|||
return CalculatedFieldType.SCRIPT; |
|||
} |
|||
|
|||
@Override |
|||
protected void validateNewEntry(ArgumentEntry newEntry) { |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { |
|||
Object[] args = ctx.getArgNames().stream() |
|||
.map(this::toTbelArgument) |
|||
.toArray(); |
|||
|
|||
ListenableFuture<JsonNode> resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args); |
|||
Output output = ctx.getOutput(); |
|||
return Futures.transform(resultFuture, |
|||
result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), |
|||
MoreExecutors.directExecutor() |
|||
); |
|||
} |
|||
|
|||
private TbelCfArg toTbelArgument(String key) { |
|||
return arguments.get(key).toTbelCfArg(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,70 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.kv.BasicKvEntry; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { |
|||
|
|||
public SimpleCalculatedFieldState(List<String> requiredArguments) { |
|||
super(requiredArguments); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedFieldType getType() { |
|||
return CalculatedFieldType.SIMPLE; |
|||
} |
|||
|
|||
@Override |
|||
protected void validateNewEntry(ArgumentEntry newEntry) { |
|||
if (newEntry instanceof TsRollingArgumentEntry) { |
|||
throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { |
|||
var expr = ctx.getCustomExpression().get(); |
|||
|
|||
for (Map.Entry<String, ArgumentEntry> entry : this.arguments.entrySet()) { |
|||
try { |
|||
BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); |
|||
expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); |
|||
} catch (NumberFormatException e) { |
|||
throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); |
|||
} |
|||
} |
|||
|
|||
double expressionResult = expr.evaluate(); |
|||
|
|||
Output output = ctx.getOutput(); |
|||
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult)))); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,115 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BasicKvEntry; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
@AllArgsConstructor |
|||
public class SingleValueArgumentEntry implements ArgumentEntry { |
|||
|
|||
private long ts; |
|||
private BasicKvEntry kvEntryValue; |
|||
private Long version; |
|||
|
|||
private boolean forceResetPrevious; |
|||
|
|||
public SingleValueArgumentEntry(TsKvProto entry) { |
|||
this.ts = entry.getTs(); |
|||
this.version = entry.getVersion(); |
|||
this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); |
|||
} |
|||
|
|||
public SingleValueArgumentEntry(AttributeValueProto entry) { |
|||
this.ts = entry.getLastUpdateTs(); |
|||
this.version = entry.getVersion(); |
|||
this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); |
|||
} |
|||
|
|||
public SingleValueArgumentEntry(KvEntry entry) { |
|||
if (entry instanceof TsKvEntry tsKvEntry) { |
|||
this.ts = tsKvEntry.getTs(); |
|||
this.version = tsKvEntry.getVersion(); |
|||
} else if (entry instanceof AttributeKvEntry attributeKvEntry) { |
|||
this.ts = attributeKvEntry.getLastUpdateTs(); |
|||
this.version = attributeKvEntry.getVersion(); |
|||
} |
|||
this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); |
|||
} |
|||
|
|||
public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { |
|||
this.ts = ts; |
|||
this.kvEntryValue = kvEntryValue; |
|||
this.version = version; |
|||
} |
|||
|
|||
@Override |
|||
public ArgumentEntryType getType() { |
|||
return ArgumentEntryType.SINGLE_VALUE; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isEmpty() { |
|||
return kvEntryValue == null; |
|||
} |
|||
|
|||
@JsonIgnore |
|||
public Object getValue() { |
|||
return isEmpty() ? null : kvEntryValue.getValue(); |
|||
} |
|||
|
|||
@Override |
|||
public TbelCfArg toTbelCfArg() { |
|||
return new TbelCfSingleValueArg(ts, kvEntryValue.getValue()); |
|||
} |
|||
|
|||
@Override |
|||
public boolean updateEntry(ArgumentEntry entry) { |
|||
if (entry instanceof SingleValueArgumentEntry singleValueEntry) { |
|||
if (singleValueEntry.getTs() == this.ts) { |
|||
return false; |
|||
} |
|||
|
|||
Long newVersion = singleValueEntry.getVersion(); |
|||
if (newVersion == null || this.version == null || newVersion > this.version) { |
|||
this.ts = singleValueEntry.getTs(); |
|||
this.version = newVersion; |
|||
BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); |
|||
if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) { |
|||
return false; |
|||
} |
|||
this.kvEntryValue = singleValueEntry.getKvEntryValue(); |
|||
return true; |
|||
} |
|||
} else { |
|||
throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); |
|||
} |
|||
return false; |
|||
} |
|||
} |
|||
@ -0,0 +1,146 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; |
|||
import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.TreeMap; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
@AllArgsConstructor |
|||
@Slf4j |
|||
public class TsRollingArgumentEntry implements ArgumentEntry { |
|||
|
|||
private Integer limit; |
|||
private Long timeWindow; |
|||
private TreeMap<Long, Double> tsRecords = new TreeMap<>(); |
|||
|
|||
private boolean forceResetPrevious; |
|||
|
|||
public TsRollingArgumentEntry(List<TsKvEntry> kvEntries, int limit, long timeWindow) { |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); |
|||
} |
|||
|
|||
public TsRollingArgumentEntry(TreeMap<Long, Double> tsRecords, int limit, long timeWindow) { |
|||
this.tsRecords = tsRecords; |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
} |
|||
|
|||
public TsRollingArgumentEntry(int limit, long timeWindow) { |
|||
this.tsRecords = new TreeMap<>(); |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
} |
|||
|
|||
public TsRollingArgumentEntry(Integer limit, Long timeWindow, TreeMap<Long, Double> tsRecords) { |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
this.tsRecords = tsRecords; |
|||
} |
|||
|
|||
@Override |
|||
public ArgumentEntryType getType() { |
|||
return ArgumentEntryType.TS_ROLLING; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isEmpty() { |
|||
return tsRecords.isEmpty(); |
|||
} |
|||
|
|||
@JsonIgnore |
|||
@Override |
|||
public Object getValue() { |
|||
return tsRecords; |
|||
} |
|||
|
|||
@Override |
|||
public TbelCfArg toTbelCfArg() { |
|||
List<TbelCfTsDoubleVal> values = new ArrayList<>(tsRecords.size()); |
|||
for (var e : tsRecords.entrySet()) { |
|||
values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); |
|||
} |
|||
return new TbelCfTsRollingArg(limit, timeWindow, values); |
|||
} |
|||
|
|||
@Override |
|||
public boolean updateEntry(ArgumentEntry entry) { |
|||
if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { |
|||
updateTsRollingEntry(tsRollingEntry); |
|||
} else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { |
|||
updateSingleValueEntry(singleValueEntry); |
|||
} else { |
|||
throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); |
|||
} |
|||
return true; |
|||
} |
|||
|
|||
private void updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { |
|||
for (Map.Entry<Long, Double> tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { |
|||
addTsRecord(tsRecordEntry.getKey(), tsRecordEntry.getValue()); |
|||
} |
|||
} |
|||
|
|||
private void updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { |
|||
addTsRecord(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); |
|||
} |
|||
|
|||
private void addTsRecord(Long ts, KvEntry value) { |
|||
try { |
|||
switch (value.getDataType()) { |
|||
case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); |
|||
case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); |
|||
case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); |
|||
case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); |
|||
case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); |
|||
} |
|||
} catch (Exception e) { |
|||
tsRecords.put(ts, Double.NaN); |
|||
log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); |
|||
} finally { |
|||
cleanupExpiredRecords(); |
|||
} |
|||
} |
|||
|
|||
private void addTsRecord(Long ts, double value) { |
|||
tsRecords.put(ts, value); |
|||
cleanupExpiredRecords(); |
|||
} |
|||
|
|||
private void cleanupExpiredRecords() { |
|||
if (tsRecords.size() > limit) { |
|||
tsRecords.pollFirstEntry(); |
|||
} |
|||
tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,106 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.entitiy.cf; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.transaction.annotation.Transactional; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.audit.ActionType; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.entitiy.AbstractTbEntityService; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
|
|||
import java.util.Optional; |
|||
|
|||
@TbCoreComponent |
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { |
|||
|
|||
private final CalculatedFieldService calculatedFieldService; |
|||
|
|||
@Override |
|||
public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { |
|||
ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; |
|||
TenantId tenantId = calculatedField.getTenantId(); |
|||
try { |
|||
if (ActionType.UPDATED.equals(actionType)) { |
|||
CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); |
|||
checkForEntityChange(existingCf, calculatedField); |
|||
} |
|||
checkEntityExistence(tenantId, calculatedField.getEntityId()); |
|||
CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); |
|||
logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); |
|||
return savedCalculatedField; |
|||
} catch (ThingsboardException e) { |
|||
logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { |
|||
return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); |
|||
} |
|||
|
|||
@Override |
|||
public PageData<CalculatedField> findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { |
|||
TenantId tenantId = user.getTenantId(); |
|||
checkEntityExistence(tenantId, entityId); |
|||
return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); |
|||
} |
|||
|
|||
@Override |
|||
@Transactional |
|||
public void delete(CalculatedField calculatedField, SecurityUser user) { |
|||
ActionType actionType = ActionType.DELETED; |
|||
TenantId tenantId = calculatedField.getTenantId(); |
|||
CalculatedFieldId calculatedFieldId = calculatedField.getId(); |
|||
try { |
|||
calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); |
|||
logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); |
|||
} catch (Exception e) { |
|||
logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { |
|||
if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { |
|||
throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); |
|||
} |
|||
} |
|||
|
|||
private void checkEntityExistence(TenantId tenantId, EntityId entityId) { |
|||
switch (entityId.getEntityType()) { |
|||
case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) |
|||
.orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); |
|||
default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.entitiy.cf; |
|||
|
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
|
|||
public interface TbCalculatedFieldService { |
|||
|
|||
CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; |
|||
|
|||
CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); |
|||
|
|||
PageData<CalculatedField> findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); |
|||
|
|||
void delete(CalculatedField calculatedField, SecurityUser user); |
|||
|
|||
} |
|||
@ -0,0 +1,43 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.housekeeper.processor; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; |
|||
import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
|
|||
@Component |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor<HousekeeperTask> { |
|||
|
|||
private final CalculatedFieldService calculatedFieldService; |
|||
|
|||
@Override |
|||
public void process(HousekeeperTask task) throws Exception { |
|||
int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); |
|||
log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); |
|||
} |
|||
|
|||
@Override |
|||
public HousekeeperTaskType getTaskType() { |
|||
return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,269 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.queue; |
|||
|
|||
import com.google.common.util.concurrent.ListeningExecutorService; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.ApplicationEventPublisher; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.ThingsBoardExecutors; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.queue.QueueConfig; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.tenant.TbTenantProfileCache; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; |
|||
import org.thingsboard.server.queue.TbQueueConsumer; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldCache; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
|||
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
|||
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
|||
import org.thingsboard.server.service.queue.processing.AbstractConsumerService; |
|||
import org.thingsboard.server.service.queue.processing.IdMsgPair; |
|||
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; |
|||
|
|||
import java.util.List; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.CountDownLatch; |
|||
import java.util.concurrent.Future; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.thingsboard.server.common.util.ProtoUtils.fromProto; |
|||
|
|||
@Service |
|||
@TbRuleEngineComponent |
|||
@Slf4j |
|||
public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService<ToCalculatedFieldNotificationMsg> implements TbCalculatedFieldConsumerService { |
|||
|
|||
@Value("${queue.calculated_fields.poll_interval:25}") |
|||
private long pollInterval; |
|||
@Value("${queue.calculated_fields.pack_processing_timeout:60000}") |
|||
private long packProcessingTimeout; |
|||
@Value("${queue.calculated_fields.pool_size:8}") |
|||
private int poolSize; |
|||
|
|||
private final TbRuleEngineQueueFactory queueFactory; |
|||
private final CalculatedFieldStateService stateService; |
|||
|
|||
private PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer; |
|||
|
|||
public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, |
|||
ActorSystemContext actorContext, |
|||
TbDeviceProfileCache deviceProfileCache, |
|||
TbAssetProfileCache assetProfileCache, |
|||
TbTenantProfileCache tenantProfileCache, |
|||
TbApiUsageStateService apiUsageStateService, |
|||
PartitionService partitionService, |
|||
ApplicationEventPublisher eventPublisher, |
|||
JwtSettingsService jwtSettingsService, |
|||
CalculatedFieldCache calculatedFieldCache, |
|||
CalculatedFieldStateService stateService) { |
|||
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, |
|||
eventPublisher, jwtSettingsService); |
|||
this.queueFactory = tbQueueFactory; |
|||
this.stateService = stateService; |
|||
} |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
super.init("tb-cf"); |
|||
|
|||
this.eventConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToCalculatedFieldMsg>>create() |
|||
.queueKey(QueueKey.CF) |
|||
.topic(partitionService.getTopic(QueueKey.CF)) |
|||
.pollInterval(pollInterval) |
|||
.msgPackProcessor(this::processMsgs) |
|||
.consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) |
|||
.consumerExecutor(consumersExecutor) |
|||
.scheduler(scheduler) |
|||
.taskExecutor(mgmtExecutor) |
|||
.build(); |
|||
stateService.init(eventConsumer); |
|||
} |
|||
|
|||
@PreDestroy |
|||
public void destroy() { |
|||
super.destroy(); |
|||
} |
|||
|
|||
@Override |
|||
protected void startConsumers() { |
|||
super.startConsumers(); |
|||
} |
|||
|
|||
@Override |
|||
protected void onTbApplicationEvent(PartitionChangeEvent event) { |
|||
var partitions = event.getCfPartitions(); |
|||
try { |
|||
stateService.restore(partitions); |
|||
// eventConsumer's partitions will be updated by stateService
|
|||
|
|||
// Cleanup old entities after corresponding consumers are stopped.
|
|||
// Any periodic tasks need to check that the entity is still managed by the current server before processing.
|
|||
actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); |
|||
} catch (Throwable t) { |
|||
log.error("Failed to process partition change event: {}", event, t); |
|||
} |
|||
} |
|||
|
|||
private boolean[] partitionsToBooleanIndexArray(Set<TopicPartitionInfo> partitions) { |
|||
boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; |
|||
for (var tpi : partitions) { |
|||
tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); |
|||
} |
|||
return myPartitions; |
|||
} |
|||
|
|||
private void processMsgs(List<TbProtoQueueMsg<ToCalculatedFieldMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldMsg>> consumer, QueueConfig config) throws Exception { |
|||
List<IdMsgPair<ToCalculatedFieldMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); |
|||
ConcurrentMap<UUID, TbProtoQueueMsg<ToCalculatedFieldMsg>> pendingMap = orderedMsgList.stream().collect( |
|||
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); |
|||
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); |
|||
TbPackProcessingContext<TbProtoQueueMsg<ToCalculatedFieldMsg>> ctx = new TbPackProcessingContext<>( |
|||
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); |
|||
PendingMsgHolder<ToCalculatedFieldMsg> pendingMsgHolder = new PendingMsgHolder<>(); |
|||
Future<?> packSubmitFuture = consumersExecutor.submit(() -> { |
|||
orderedMsgList.forEach((element) -> { |
|||
UUID id = element.getUuid(); |
|||
TbProtoQueueMsg<ToCalculatedFieldMsg> msg = element.getMsg(); |
|||
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); |
|||
TbCallback callback = new TbPackCallback<>(id, ctx); |
|||
try { |
|||
ToCalculatedFieldMsg toCfMsg = msg.getValue(); |
|||
pendingMsgHolder.setMsg(toCfMsg); |
|||
if (toCfMsg.hasTelemetryMsg()) { |
|||
log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); |
|||
forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); |
|||
} else if (toCfMsg.hasLinkedTelemetryMsg()) { |
|||
forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); |
|||
} else if (toCfMsg.hasComponentLifecycleMsg()) { |
|||
log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); |
|||
forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); |
|||
} |
|||
} catch (Throwable e) { |
|||
log.warn("[{}] Failed to process message: {}", id, msg, e); |
|||
callback.onFailure(e); |
|||
} |
|||
}); |
|||
}); |
|||
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { |
|||
if (!packSubmitFuture.isDone()) { |
|||
packSubmitFuture.cancel(true); |
|||
log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); |
|||
} |
|||
if (log.isDebugEnabled()) { |
|||
ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); |
|||
} |
|||
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); |
|||
} |
|||
consumer.commit(); |
|||
} |
|||
|
|||
@Override |
|||
protected ServiceType getServiceType() { |
|||
return ServiceType.TB_RULE_ENGINE; |
|||
} |
|||
|
|||
@Override |
|||
protected long getNotificationPollDuration() { |
|||
return pollInterval; |
|||
} |
|||
|
|||
@Override |
|||
protected long getNotificationPackProcessingTimeout() { |
|||
return packProcessingTimeout; |
|||
} |
|||
|
|||
@Override |
|||
protected int getMgmtThreadPoolSize() { |
|||
return Math.max(Runtime.getRuntime().availableProcessors(), 4); |
|||
} |
|||
|
|||
@Override |
|||
protected TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldNotificationMsg>> createNotificationsConsumer() { |
|||
return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); |
|||
} |
|||
|
|||
@Override |
|||
protected void handleNotification(UUID id, TbProtoQueueMsg<ToCalculatedFieldNotificationMsg> msg, TbCallback callback) { |
|||
ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); |
|||
if (toCfNotification.hasComponentLifecycleMsg()) { |
|||
// from upstream (maybe removed since we don't need to init state for each partition)
|
|||
log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); |
|||
forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); |
|||
} else if (toCfNotification.hasLinkedTelemetryMsg()) { |
|||
forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); |
|||
} |
|||
} |
|||
|
|||
private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { |
|||
var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); |
|||
var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); |
|||
actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); |
|||
} |
|||
|
|||
private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { |
|||
var msg = linkedMsg.getMsg(); |
|||
var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); |
|||
var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); |
|||
actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); |
|||
} |
|||
|
|||
private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { |
|||
var msg = fromProto(proto); |
|||
actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); |
|||
} |
|||
|
|||
private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { |
|||
return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); |
|||
} |
|||
|
|||
@Override |
|||
protected void stopConsumers() { |
|||
super.stopConsumers(); |
|||
eventConsumer.stop(); |
|||
eventConsumer.awaitStop(); |
|||
stateService.stop(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.queue; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.Setter; |
|||
|
|||
public class PendingMsgHolder<T> { |
|||
@Getter @Setter |
|||
private volatile T msg; |
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.queue; |
|||
|
|||
import org.springframework.context.ApplicationListener; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
|
|||
public interface TbCalculatedFieldConsumerService extends ApplicationListener<PartitionChangeEvent> { |
|||
|
|||
} |
|||
@ -0,0 +1,152 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.utils; |
|||
|
|||
import org.thingsboard.server.common.data.StringUtils; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.BasicKvEntry; |
|||
import org.thingsboard.server.common.util.KvProtoUtil; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; |
|||
|
|||
import java.util.Optional; |
|||
import java.util.TreeMap; |
|||
import java.util.UUID; |
|||
|
|||
public class CalculatedFieldUtils { |
|||
|
|||
public static CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { |
|||
return CalculatedFieldIdProto.newBuilder() |
|||
.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) |
|||
.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) |
|||
.build(); |
|||
} |
|||
|
|||
public static CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { |
|||
return CalculatedFieldEntityCtxIdProto.newBuilder() |
|||
.setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) |
|||
.setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) |
|||
.setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) |
|||
.setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) |
|||
.setEntityType(ctxId.entityId().getEntityType().name()) |
|||
.setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) |
|||
.setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) |
|||
.build(); |
|||
} |
|||
|
|||
public static CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { |
|||
TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); |
|||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); |
|||
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); |
|||
return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); |
|||
} |
|||
|
|||
public static CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { |
|||
CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() |
|||
.setId(toProto(stateId)) |
|||
.setType(state.getType().name()); |
|||
|
|||
state.getArguments().forEach((argName, argEntry) -> { |
|||
if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { |
|||
builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); |
|||
} else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { |
|||
builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); |
|||
} |
|||
}); |
|||
return builder.build(); |
|||
} |
|||
|
|||
public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { |
|||
SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() |
|||
.setArgName(argName); |
|||
|
|||
if (entry.getKvEntryValue() != null) { |
|||
builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); |
|||
} |
|||
|
|||
Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); |
|||
|
|||
return builder.build(); |
|||
} |
|||
|
|||
public static TsRollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { |
|||
TsRollingArgumentProto.Builder builder = TsRollingArgumentProto.newBuilder() |
|||
.setKey(argName) |
|||
.setLimit(entry.getLimit()) |
|||
.setTimeWindow(entry.getTimeWindow()); |
|||
|
|||
entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(TsDoubleValProto.newBuilder().setTs(ts).setValue(value).build())); |
|||
|
|||
return builder.build(); |
|||
} |
|||
|
|||
public static CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { |
|||
if (StringUtils.isEmpty(proto.getType())) { |
|||
return null; |
|||
} |
|||
|
|||
CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); |
|||
|
|||
CalculatedFieldState state = switch (type) { |
|||
case SIMPLE -> new SimpleCalculatedFieldState(); |
|||
case SCRIPT -> new ScriptCalculatedFieldState(); |
|||
}; |
|||
|
|||
proto.getSingleValueArgumentsList().forEach(argProto -> |
|||
state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); |
|||
|
|||
if (CalculatedFieldType.SCRIPT.equals(type)) { |
|||
proto.getRollingValueArgumentsList().forEach(argProto -> |
|||
state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); |
|||
} |
|||
|
|||
return state; |
|||
} |
|||
|
|||
public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { |
|||
if (!proto.hasValue()) { |
|||
return new SingleValueArgumentEntry(); |
|||
} |
|||
TsValueProto tsValueProto = proto.getValue(); |
|||
return new SingleValueArgumentEntry( |
|||
tsValueProto.getTs(), |
|||
(BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto), |
|||
proto.getVersion() |
|||
); |
|||
} |
|||
|
|||
public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) { |
|||
TreeMap<Long, Double> tsRecords = new TreeMap<>(); |
|||
proto.getTsValueList().forEach(tsValueProto -> tsRecords.put(tsValueProto.getTs(), tsValueProto.getValue())); |
|||
return new TsRollingArgumentEntry(tsRecords, proto.getLimit(), proto.getTimeWindow()); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.utils; |
|||
|
|||
import lombok.Data; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
@Component |
|||
@Data |
|||
public class DebugModeRateLimitsConfig { |
|||
|
|||
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") |
|||
private boolean ruleChainDebugPerTenantLimitsEnabled; |
|||
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") |
|||
private String ruleChainDebugPerTenantLimitsConfiguration; |
|||
|
|||
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") |
|||
private boolean calculatedFieldDebugPerTenantLimitsEnabled; |
|||
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") |
|||
private String calculatedFieldDebugPerTenantLimitsConfiguration; |
|||
|
|||
} |
|||
@ -0,0 +1,71 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.utils; |
|||
|
|||
import lombok.SneakyThrows; |
|||
import org.rocksdb.Options; |
|||
import org.rocksdb.RocksDB; |
|||
import org.rocksdb.RocksIterator; |
|||
import org.rocksdb.WriteOptions; |
|||
|
|||
import java.nio.charset.StandardCharsets; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.util.function.BiConsumer; |
|||
|
|||
public class TbRocksDb { |
|||
|
|||
protected final String path; |
|||
private final WriteOptions writeOptions; |
|||
protected final RocksDB db; |
|||
|
|||
static { |
|||
RocksDB.loadLibrary(); |
|||
} |
|||
|
|||
public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) throws Exception { |
|||
this.path = path; |
|||
this.writeOptions = writeOptions; |
|||
Files.createDirectories(Path.of(path).getParent()); |
|||
this.db = RocksDB.open(dbOptions, path); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public void put(String key, byte[] value) { |
|||
db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); |
|||
} |
|||
|
|||
public void forEach(BiConsumer<String, byte[]> processor) { |
|||
try (RocksIterator iterator = db.newIterator()) { |
|||
for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { |
|||
String key = new String(iterator.key(), StandardCharsets.UTF_8); |
|||
processor.accept(key, iterator.value()); |
|||
} |
|||
} |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public void delete(String key) { |
|||
db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); |
|||
} |
|||
|
|||
public void close() { |
|||
if (db != null) { |
|||
db.close(); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,459 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.cf; |
|||
|
|||
import com.fasterxml.jackson.databind.node.ArrayNode; |
|||
import com.fasterxml.jackson.databind.node.ObjectNode; |
|||
import org.junit.Test; |
|||
import org.junit.jupiter.api.BeforeEach; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.DataConstants; |
|||
import org.thingsboard.server.common.data.Device; |
|||
import org.thingsboard.server.common.data.asset.Asset; |
|||
import org.thingsboard.server.common.data.asset.AssetProfile; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.debug.DebugSettings; |
|||
import org.thingsboard.server.common.data.id.AssetProfileId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.controller.CalculatedFieldControllerTest; |
|||
import org.thingsboard.server.dao.service.DaoSqlTest; |
|||
|
|||
import java.util.Map; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.awaitility.Awaitility.await; |
|||
|
|||
@DaoSqlTest |
|||
public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { |
|||
|
|||
@BeforeEach |
|||
void setUp() throws Exception { |
|||
loginTenantAdmin(); |
|||
} |
|||
|
|||
@Test |
|||
public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); |
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); |
|||
|
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setEntityId(testDevice.getId()); |
|||
calculatedField.setType(CalculatedFieldType.SIMPLE); |
|||
calculatedField.setName("C to F"); |
|||
calculatedField.setDebugSettings(DebugSettings.all()); |
|||
calculatedField.setConfigurationVersion(1); |
|||
|
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument = new Argument(); |
|||
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); |
|||
argument.setRefEntityKey(refEntityKey); |
|||
argument.setDefaultValue("12"); // not used because real telemetry value in db is present
|
|||
config.setArguments(Map.of("T", argument)); |
|||
config.setExpression("(T * 9/5) + 32"); |
|||
|
|||
Output output = new Output(); |
|||
output.setName("fahrenheitTemp"); |
|||
output.setType(OutputType.TIME_SERIES); |
|||
config.setOutput(output); |
|||
|
|||
calculatedField.setConfiguration(config); |
|||
calculatedField.setVersion(1L); |
|||
|
|||
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
|
|||
await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); |
|||
|
|||
await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); |
|||
}); |
|||
|
|||
Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); |
|||
savedOutput.setType(OutputType.ATTRIBUTES); |
|||
savedOutput.setScope(AttributeScope.SERVER_SCOPE); |
|||
savedOutput.setName("temperatureF"); |
|||
savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); |
|||
|
|||
await().alias("update CF output -> perform calculation with updated output").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); |
|||
assertThat(temperatureF).isNotNull(); |
|||
assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); |
|||
}); |
|||
|
|||
Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); |
|||
savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); |
|||
savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); |
|||
|
|||
await().alias("update CF argument -> perform calculation with new argument").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); |
|||
assertThat(temperatureF).isNotNull(); |
|||
assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); |
|||
}); |
|||
|
|||
savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); |
|||
savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); |
|||
|
|||
await().alias("update CF expression -> perform calculation with new expression").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); |
|||
assertThat(temperatureF).isNotNull(); |
|||
assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); |
|||
}); |
|||
} |
|||
|
|||
@Test |
|||
public void testSimpleCalculatedFieldWhenNotAllTelemetryPresent() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
|
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setEntityId(testDevice.getId()); |
|||
calculatedField.setType(CalculatedFieldType.SIMPLE); |
|||
calculatedField.setName("C to F"); |
|||
calculatedField.setDebugSettings(DebugSettings.all()); |
|||
calculatedField.setConfigurationVersion(1); |
|||
|
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument = new Argument(); |
|||
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); |
|||
argument.setRefEntityKey(refEntityKey); |
|||
config.setArguments(Map.of("T", argument)); |
|||
config.setExpression("(T * 9/5) + 32"); |
|||
|
|||
Output output = new Output(); |
|||
output.setName("fahrenheitTemp"); |
|||
output.setType(OutputType.TIME_SERIES); |
|||
config.setOutput(output); |
|||
|
|||
calculatedField.setConfiguration(config); |
|||
calculatedField.setVersion(1L); |
|||
|
|||
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
|
|||
await().alias("create CF -> state is not ready -> no calculation performed").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); |
|||
|
|||
await().alias("update telemetry -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); |
|||
}); |
|||
} |
|||
|
|||
@Test |
|||
public void testSimpleCalculatedFieldWhenNotAllTelemetryPresentButDefaultValueIsSet() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
|
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setEntityId(testDevice.getId()); |
|||
calculatedField.setType(CalculatedFieldType.SIMPLE); |
|||
calculatedField.setName("C to F"); |
|||
calculatedField.setDebugSettings(DebugSettings.all()); |
|||
calculatedField.setConfigurationVersion(1); |
|||
|
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument = new Argument(); |
|||
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); |
|||
argument.setRefEntityKey(refEntityKey); |
|||
argument.setDefaultValue("12"); |
|||
config.setArguments(Map.of("T", argument)); |
|||
config.setExpression("(T * 9/5) + 32"); |
|||
|
|||
Output output = new Output(); |
|||
output.setName("fahrenheitTemp"); |
|||
output.setType(OutputType.TIME_SERIES); |
|||
config.setOutput(output); |
|||
|
|||
calculatedField.setConfiguration(config); |
|||
calculatedField.setVersion(1L); |
|||
|
|||
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
|
|||
await().alias("create CF -> perform initial calculation with default value").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); |
|||
|
|||
await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); |
|||
}); |
|||
} |
|||
|
|||
@Test |
|||
public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":40}")); |
|||
|
|||
AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); |
|||
|
|||
Asset asset1 = createAsset("Test asset 1", assetProfile.getId()); |
|||
doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":11}")); |
|||
|
|||
Asset asset2 = createAsset("Test asset 2", assetProfile.getId()); |
|||
doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":12}")); |
|||
|
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setEntityId(assetProfile.getId()); |
|||
calculatedField.setType(CalculatedFieldType.SIMPLE); |
|||
calculatedField.setName("z = x + y"); |
|||
calculatedField.setDebugSettings(DebugSettings.all()); |
|||
calculatedField.setConfigurationVersion(1); |
|||
|
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument1 = new Argument(); |
|||
ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("y", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); |
|||
argument1.setRefEntityKey(refEntityKey1); |
|||
|
|||
Argument argument2 = new Argument(); |
|||
argument2.setRefEntityId(testDevice.getId()); |
|||
ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("x", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); |
|||
argument2.setRefEntityKey(refEntityKey2); |
|||
|
|||
config.setArguments(Map.of("x", argument2, "y", argument1)); |
|||
|
|||
config.setExpression("x + y"); |
|||
|
|||
Output output = new Output(); |
|||
output.setName("z"); |
|||
output.setType(OutputType.ATTRIBUTES); |
|||
output.setScope(AttributeScope.SERVER_SCOPE); |
|||
|
|||
config.setOutput(output); |
|||
|
|||
calculatedField.setConfiguration(config); |
|||
calculatedField.setVersion(1L); |
|||
|
|||
doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
|
|||
await().alias("create CF and perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
// result of asset 1
|
|||
ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); |
|||
assertThat(z1).isNotNull(); |
|||
assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); |
|||
|
|||
// result of asset 2
|
|||
ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); |
|||
assertThat(z2).isNotNull(); |
|||
assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); |
|||
|
|||
await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
// result of asset 1
|
|||
ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); |
|||
assertThat(z1).isNotNull(); |
|||
assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); |
|||
|
|||
// result of asset 2
|
|||
ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); |
|||
assertThat(z2).isNotNull(); |
|||
assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); |
|||
|
|||
await().alias("update asset 1 telemetry -> recalculate state only for asset 1").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
// result of asset 1
|
|||
ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); |
|||
assertThat(z1).isNotNull(); |
|||
assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); |
|||
|
|||
// result of asset 2 (no changes)
|
|||
ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); |
|||
assertThat(z2).isNotNull(); |
|||
assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); |
|||
|
|||
await().alias("update asset 2 telemetry -> recalculate state only for asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
// result of asset 1 (no changes)
|
|||
ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); |
|||
assertThat(z1).isNotNull(); |
|||
assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); |
|||
|
|||
// result of asset 2
|
|||
ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); |
|||
assertThat(z2).isNotNull(); |
|||
assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); |
|||
}); |
|||
|
|||
Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); |
|||
doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); |
|||
|
|||
Asset finalAsset3 = asset3; |
|||
await().alias("add new entity to profile -> calculate state for new entity").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
// result of asset 3
|
|||
ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); |
|||
assertThat(z3).isNotNull(); |
|||
assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); |
|||
|
|||
await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
// result of asset 1
|
|||
ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); |
|||
assertThat(z1).isNotNull(); |
|||
assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); |
|||
|
|||
// result of asset 2
|
|||
ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); |
|||
assertThat(z2).isNotNull(); |
|||
assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); |
|||
|
|||
// result of asset 3
|
|||
ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); |
|||
assertThat(z3).isNotNull(); |
|||
assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); |
|||
}); |
|||
|
|||
// update profile for asset 3 -> delete state for asset 3
|
|||
AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); |
|||
asset3.setAssetProfileId(newAssetProfile.getId()); |
|||
asset3 = doPost("/api/asset", asset3, Asset.class); |
|||
|
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); |
|||
|
|||
Asset updatedAsset3 = asset3; |
|||
await().alias("update device telemetry -> recalculate state for asset 1 and asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
// result of asset 1
|
|||
ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); |
|||
assertThat(z1).isNotNull(); |
|||
assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); |
|||
|
|||
// result of asset 2
|
|||
ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); |
|||
assertThat(z2).isNotNull(); |
|||
assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); |
|||
|
|||
// no changes for asset 3
|
|||
ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z"); |
|||
assertThat(z3).isNotNull(); |
|||
assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); |
|||
}); |
|||
} |
|||
|
|||
@Test |
|||
public void testSimpleCalculatedFieldWhenExpressionIsInvalid() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); |
|||
|
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setEntityId(testDevice.getId()); |
|||
calculatedField.setType(CalculatedFieldType.SIMPLE); |
|||
calculatedField.setName("C to F"); |
|||
calculatedField.setDebugSettings(DebugSettings.all()); |
|||
calculatedField.setConfigurationVersion(1); |
|||
|
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument = new Argument(); |
|||
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); |
|||
argument.setRefEntityKey(refEntityKey); |
|||
argument.setDefaultValue("12"); // not used because real telemetry value in db is present
|
|||
config.setArguments(Map.of("T", argument)); |
|||
config.setExpression("(T * 9/0) + 32"); |
|||
|
|||
Output output = new Output(); |
|||
output.setName("fahrenheitTemp"); |
|||
output.setType(OutputType.TIME_SERIES); |
|||
config.setOutput(output); |
|||
|
|||
calculatedField.setConfiguration(config); |
|||
calculatedField.setVersion(1L); |
|||
|
|||
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
|
|||
await().alias("create CF -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); |
|||
}); |
|||
|
|||
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); |
|||
|
|||
await().alias("update telemetry -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.untilAsserted(() -> { |
|||
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); |
|||
assertThat(fahrenheitTemp).isNotNull(); |
|||
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); |
|||
}); |
|||
} |
|||
|
|||
private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { |
|||
return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); |
|||
} |
|||
|
|||
private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { |
|||
return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); |
|||
} |
|||
|
|||
private Asset createAsset(String name, AssetProfileId assetProfileId) { |
|||
Asset asset = new Asset(); |
|||
asset.setName(name); |
|||
asset.setAssetProfileId(assetProfileId); |
|||
return doPost("/api/asset", asset, Asset.class); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,163 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import org.junit.After; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
import org.thingsboard.server.common.data.Device; |
|||
import org.thingsboard.server.common.data.Tenant; |
|||
import org.thingsboard.server.common.data.User; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.security.Authority; |
|||
import org.thingsboard.server.dao.service.DaoSqlTest; |
|||
|
|||
import java.util.Map; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; |
|||
|
|||
@DaoSqlTest |
|||
public class CalculatedFieldControllerTest extends AbstractControllerTest { |
|||
|
|||
private Tenant savedTenant; |
|||
|
|||
@Before |
|||
public void beforeTest() throws Exception { |
|||
loginSysAdmin(); |
|||
|
|||
Tenant tenant = new Tenant(); |
|||
tenant.setTitle("My tenant"); |
|||
savedTenant = saveTenant(tenant); |
|||
assertThat(savedTenant).isNotNull(); |
|||
|
|||
User tenantAdmin = new User(); |
|||
tenantAdmin.setAuthority(Authority.TENANT_ADMIN); |
|||
tenantAdmin.setTenantId(savedTenant.getId()); |
|||
tenantAdmin.setEmail("tenant2@thingsboard.org"); |
|||
tenantAdmin.setFirstName("Joe"); |
|||
tenantAdmin.setLastName("Downs"); |
|||
|
|||
createUserAndLogin(tenantAdmin, "testPassword1"); |
|||
} |
|||
|
|||
@After |
|||
public void afterTest() throws Exception { |
|||
loginSysAdmin(); |
|||
|
|||
deleteTenant(savedTenant.getId()); |
|||
} |
|||
|
|||
@Test |
|||
public void testSaveCalculatedField() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
CalculatedField calculatedField = getCalculatedField(testDevice.getId()); |
|||
|
|||
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
|
|||
assertThat(savedCalculatedField).isNotNull(); |
|||
assertThat(savedCalculatedField.getId()).isNotNull(); |
|||
assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); |
|||
assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); |
|||
assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); |
|||
assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); |
|||
assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); |
|||
assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getCalculatedFieldConfig(testDevice.getId())); |
|||
assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); |
|||
|
|||
savedCalculatedField.setName("Test CF"); |
|||
|
|||
CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); |
|||
|
|||
assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); |
|||
assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); |
|||
|
|||
doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) |
|||
.andExpect(status().isOk()); |
|||
} |
|||
|
|||
@Test |
|||
public void testGetCalculatedFieldById() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
CalculatedField calculatedField = getCalculatedField(testDevice.getId()); |
|||
|
|||
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); |
|||
|
|||
assertThat(fetchedCalculatedField).isNotNull(); |
|||
assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); |
|||
|
|||
doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) |
|||
.andExpect(status().isOk()); |
|||
} |
|||
|
|||
@Test |
|||
public void testDeleteCalculatedField() throws Exception { |
|||
Device testDevice = createDevice("Test device", "1234567890"); |
|||
CalculatedField calculatedField = getCalculatedField(testDevice.getId()); |
|||
|
|||
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); |
|||
|
|||
assertThat(savedCalculatedField).isNotNull(); |
|||
|
|||
doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) |
|||
.andExpect(status().isOk()); |
|||
doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); |
|||
} |
|||
|
|||
private CalculatedField getCalculatedField(DeviceId deviceId) { |
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setEntityId(deviceId); |
|||
calculatedField.setType(CalculatedFieldType.SIMPLE); |
|||
calculatedField.setName("Test Calculated Field"); |
|||
calculatedField.setConfigurationVersion(1); |
|||
calculatedField.setConfiguration(getCalculatedFieldConfig(null)); |
|||
calculatedField.setVersion(1L); |
|||
return calculatedField; |
|||
} |
|||
|
|||
private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { |
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument = new Argument(); |
|||
argument.setRefEntityId(referencedEntityId); |
|||
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); |
|||
argument.setRefEntityKey(refEntityKey); |
|||
|
|||
config.setArguments(Map.of("T", argument)); |
|||
|
|||
config.setExpression("T - (100 - H) / 5"); |
|||
|
|||
Output output = new Output(); |
|||
output.setName("output"); |
|||
output.setType(OutputType.TIME_SERIES); |
|||
|
|||
config.setOutput(output); |
|||
|
|||
return config; |
|||
} |
|||
|
|||
} |
|||
File diff suppressed because one or more lines are too long
@ -0,0 +1,205 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import org.junit.jupiter.api.BeforeEach; |
|||
import org.junit.jupiter.api.Test; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.boot.test.context.SpringBootTest; |
|||
import org.springframework.boot.test.mock.mockito.MockBean; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.id.AssetId; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.LongDataEntry; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
|
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
import java.util.TreeMap; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutionException; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.mockito.ArgumentMatchers.any; |
|||
import static org.mockito.Mockito.when; |
|||
|
|||
@SpringBootTest(classes = DefaultTbelInvokeService.class) |
|||
public class ScriptCalculatedFieldStateTest { |
|||
|
|||
private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); |
|||
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); |
|||
private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); |
|||
|
|||
private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L); |
|||
private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); |
|||
|
|||
private final long ts = System.currentTimeMillis(); |
|||
|
|||
private ScriptCalculatedFieldState state; |
|||
private CalculatedFieldCtx ctx; |
|||
|
|||
@Autowired |
|||
private TbelInvokeService tbelInvokeService; |
|||
|
|||
@MockBean |
|||
private ApiLimitService apiLimitService; |
|||
|
|||
@BeforeEach |
|||
void setUp() { |
|||
when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); |
|||
ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService); |
|||
ctx.init(); |
|||
state = new ScriptCalculatedFieldState(ctx.getArgNames()); |
|||
} |
|||
|
|||
@Test |
|||
void testType() { |
|||
assertThat(state.getType()).isEqualTo(CalculatedFieldType.SCRIPT); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateState() { |
|||
state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); |
|||
|
|||
Map<String, ArgumentEntry> newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); |
|||
boolean stateUpdated = state.updateState(newArgs); |
|||
|
|||
assertThat(stateUpdated).isTrue(); |
|||
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( |
|||
Map.of( |
|||
"assetHumidity", assetHumidityArgEntry, |
|||
"deviceTemperature", deviceTemperatureArgEntry |
|||
) |
|||
); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateStateWhenUpdateExistingEntry() { |
|||
state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); |
|||
|
|||
SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); |
|||
Map<String, ArgumentEntry> newArgs = Map.of("assetHumidity", newArgEntry); |
|||
boolean stateUpdated = state.updateState(newArgs); |
|||
|
|||
assertThat(stateUpdated).isTrue(); |
|||
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( |
|||
Map.of( |
|||
"assetHumidity", newArgEntry, |
|||
"deviceTemperature", deviceTemperatureArgEntry |
|||
) |
|||
); |
|||
} |
|||
|
|||
@Test |
|||
void testPerformCalculation() throws ExecutionException, InterruptedException { |
|||
state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); |
|||
|
|||
CalculatedFieldResult result = state.performCalculation(ctx).get(); |
|||
|
|||
assertThat(result).isNotNull(); |
|||
Output output = getCalculatedFieldConfig().getOutput(); |
|||
assertThat(result.getType()).isEqualTo(output.getType()); |
|||
assertThat(result.getScope()).isEqualTo(output.getScope()); |
|||
assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0))); |
|||
} |
|||
|
|||
@Test |
|||
void testIsReadyWhenNotAllArgPresent() { |
|||
assertThat(state.isReady()).isFalse(); |
|||
} |
|||
|
|||
@Test |
|||
void testIsReadyWhenAllArgPresent() { |
|||
state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); |
|||
|
|||
assertThat(state.isReady()).isTrue(); |
|||
} |
|||
|
|||
@Test |
|||
void testIsReadyWhenEmptyEntryPresents() { |
|||
state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); |
|||
|
|||
assertThat(state.isReady()).isFalse(); |
|||
} |
|||
|
|||
private TsRollingArgumentEntry createRollingArgEntry() { |
|||
TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(5, 30000L); |
|||
long ts = System.currentTimeMillis(); |
|||
|
|||
TreeMap<Long, Double> values = new TreeMap<>(); |
|||
values.put(ts - 40, 10.0); |
|||
values.put(ts - 30, 12.0); |
|||
values.put(ts - 20, 17.0); |
|||
|
|||
argumentEntry.setTsRecords(values); |
|||
return argumentEntry; |
|||
} |
|||
|
|||
private CalculatedField getCalculatedField() { |
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setTenantId(TENANT_ID); |
|||
calculatedField.setEntityId(ASSET_ID); |
|||
calculatedField.setType(CalculatedFieldType.SCRIPT); |
|||
calculatedField.setName("Test Calculated Field"); |
|||
calculatedField.setConfigurationVersion(1); |
|||
calculatedField.setConfiguration(getCalculatedFieldConfig()); |
|||
calculatedField.setVersion(1L); |
|||
return calculatedField; |
|||
} |
|||
|
|||
private CalculatedFieldConfiguration getCalculatedFieldConfig() { |
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument1 = new Argument(); |
|||
argument1.setRefEntityId(DEVICE_ID); |
|||
ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); |
|||
argument1.setRefEntityKey(refEntityKey1); |
|||
argument1.setLimit(5); |
|||
argument1.setTimeWindow(30000L); |
|||
|
|||
Argument argument2 = new Argument(); |
|||
ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); |
|||
argument1.setRefEntityKey(refEntityKey2); |
|||
|
|||
config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); |
|||
|
|||
config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}"); |
|||
|
|||
Output output = new Output(); |
|||
output.setType(OutputType.ATTRIBUTES); |
|||
output.setScope(AttributeScope.SERVER_SCOPE); |
|||
|
|||
config.setOutput(output); |
|||
|
|||
return config; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,228 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import org.junit.jupiter.api.BeforeEach; |
|||
import org.junit.jupiter.api.Test; |
|||
import org.junit.jupiter.api.extension.ExtendWith; |
|||
import org.mockito.Mock; |
|||
import org.mockito.junit.jupiter.MockitoExtension; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.id.AssetId; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.LongDataEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
|
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutionException; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.assertj.core.api.Assertions.assertThatThrownBy; |
|||
import static org.mockito.ArgumentMatchers.any; |
|||
import static org.mockito.Mockito.when; |
|||
|
|||
@ExtendWith(MockitoExtension.class) |
|||
public class SimpleCalculatedFieldStateTest { |
|||
|
|||
private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); |
|||
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); |
|||
private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); |
|||
|
|||
private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("key1", 11L), 145L); |
|||
private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new LongDataEntry("key2", 15L), 165L); |
|||
private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new LongDataEntry("key3", 23L), 184L); |
|||
|
|||
private SimpleCalculatedFieldState state; |
|||
private CalculatedFieldCtx ctx; |
|||
|
|||
@Mock |
|||
private ApiLimitService apiLimitService; |
|||
|
|||
@BeforeEach |
|||
void setUp() { |
|||
when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); |
|||
ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService); |
|||
ctx.init(); |
|||
state = new SimpleCalculatedFieldState(ctx.getArgNames()); |
|||
} |
|||
|
|||
@Test |
|||
void testType() { |
|||
assertThat(state.getType()).isEqualTo(CalculatedFieldType.SIMPLE); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateState() { |
|||
state.arguments = new HashMap<>(Map.of( |
|||
"key1", key1ArgEntry, |
|||
"key2", key2ArgEntry |
|||
)); |
|||
|
|||
Map<String, ArgumentEntry> newArgs = Map.of("key3", key3ArgEntry); |
|||
boolean stateUpdated = state.updateState(newArgs); |
|||
|
|||
assertThat(stateUpdated).isTrue(); |
|||
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( |
|||
Map.of( |
|||
"key1", key1ArgEntry, |
|||
"key2", key2ArgEntry, |
|||
"key3", key3ArgEntry |
|||
) |
|||
); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateStateWhenUpdateExistingEntry() { |
|||
state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); |
|||
|
|||
SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); |
|||
Map<String, ArgumentEntry> newArgs = Map.of("key1", newArgEntry); |
|||
boolean stateUpdated = state.updateState(newArgs); |
|||
|
|||
assertThat(stateUpdated).isTrue(); |
|||
assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateStateWhenRollingEntryPassed() { |
|||
state.arguments = new HashMap<>(Map.of( |
|||
"key1", key1ArgEntry, |
|||
"key2", key2ArgEntry |
|||
)); |
|||
|
|||
Map<String, ArgumentEntry> newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); |
|||
assertThatThrownBy(() -> state.updateState(newArgs)) |
|||
.isInstanceOf(IllegalArgumentException.class) |
|||
.hasMessage("Rolling argument entry is not supported for simple calculated fields."); |
|||
} |
|||
|
|||
@Test |
|||
void testPerformCalculation() throws ExecutionException, InterruptedException { |
|||
state.arguments = new HashMap<>(Map.of( |
|||
"key1", key1ArgEntry, |
|||
"key2", key2ArgEntry, |
|||
"key3", key3ArgEntry |
|||
)); |
|||
|
|||
CalculatedFieldResult result = state.performCalculation(ctx).get(); |
|||
|
|||
assertThat(result).isNotNull(); |
|||
Output output = getCalculatedFieldConfig().getOutput(); |
|||
assertThat(result.getType()).isEqualTo(output.getType()); |
|||
assertThat(result.getScope()).isEqualTo(output.getScope()); |
|||
assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0))); |
|||
} |
|||
|
|||
@Test |
|||
void testPerformCalculationWhenPassedNotNumber() { |
|||
state.arguments = new HashMap<>(Map.of( |
|||
"key1", key1ArgEntry, |
|||
"key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), |
|||
"key3", key3ArgEntry |
|||
)); |
|||
|
|||
assertThatThrownBy(() -> state.performCalculation(ctx)) |
|||
.isInstanceOf(IllegalArgumentException.class) |
|||
.hasMessage("Argument 'key2' is not a number."); |
|||
} |
|||
|
|||
@Test |
|||
void testIsReadyWhenNotAllArgPresent() { |
|||
assertThat(state.isReady()).isFalse(); |
|||
} |
|||
|
|||
@Test |
|||
void testIsReadyWhenAllArgPresent() { |
|||
state.arguments = new HashMap<>(Map.of( |
|||
"key1", key1ArgEntry, |
|||
"key2", key2ArgEntry, |
|||
"key3", key3ArgEntry |
|||
)); |
|||
|
|||
assertThat(state.isReady()).isTrue(); |
|||
} |
|||
|
|||
@Test |
|||
void testIsReadyWhenEmptyEntryPresents() { |
|||
state.arguments = new HashMap<>(Map.of( |
|||
"key1", key1ArgEntry, |
|||
"key2", key2ArgEntry |
|||
)); |
|||
state.getArguments().put("key3", new SingleValueArgumentEntry()); |
|||
|
|||
assertThat(state.isReady()).isFalse(); |
|||
} |
|||
|
|||
private CalculatedField getCalculatedField() { |
|||
CalculatedField calculatedField = new CalculatedField(); |
|||
calculatedField.setTenantId(TENANT_ID); |
|||
calculatedField.setEntityId(DEVICE_ID); |
|||
calculatedField.setType(CalculatedFieldType.SIMPLE); |
|||
calculatedField.setName("Test Calculated Field"); |
|||
calculatedField.setConfigurationVersion(1); |
|||
calculatedField.setConfiguration(getCalculatedFieldConfig()); |
|||
calculatedField.setVersion(1L); |
|||
return calculatedField; |
|||
} |
|||
|
|||
private CalculatedFieldConfiguration getCalculatedFieldConfig() { |
|||
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); |
|||
|
|||
Argument argument1 = new Argument(); |
|||
argument1.setRefEntityId(ASSET_ID); |
|||
ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temp1", ArgumentType.TS_LATEST, null); |
|||
argument1.setRefEntityKey(refEntityKey1); |
|||
|
|||
Argument argument2 = new Argument(); |
|||
argument2.setRefEntityId(ASSET_ID); |
|||
ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("temp2", ArgumentType.ATTRIBUTE, null); |
|||
argument2.setRefEntityKey(refEntityKey2); |
|||
|
|||
Argument argument3 = new Argument(); |
|||
argument3.setRefEntityId(ASSET_ID); |
|||
ReferencedEntityKey refEntityKey3 = new ReferencedEntityKey("temp3", ArgumentType.TS_LATEST, null); |
|||
argument3.setRefEntityKey(refEntityKey3); |
|||
|
|||
config.setArguments(Map.of("key1", argument1, "key2", argument2, "key3", argument3)); |
|||
|
|||
config.setExpression("key1 + key2 + key3"); |
|||
|
|||
Output output = new Output(); |
|||
output.setName("output"); |
|||
output.setType(OutputType.ATTRIBUTES); |
|||
output.setScope(AttributeScope.SERVER_SCOPE); |
|||
|
|||
config.setOutput(output); |
|||
|
|||
return config; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,76 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import org.junit.jupiter.api.BeforeEach; |
|||
import org.junit.jupiter.api.Test; |
|||
import org.thingsboard.server.common.data.kv.LongDataEntry; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.assertj.core.api.Assertions.assertThatThrownBy; |
|||
|
|||
public class SingleValueArgumentEntryTest { |
|||
|
|||
private SingleValueArgumentEntry entry; |
|||
|
|||
private final long ts = System.currentTimeMillis(); |
|||
|
|||
@BeforeEach |
|||
void setUp() { |
|||
entry = new SingleValueArgumentEntry(ts, new LongDataEntry("key", 11L), 363L); |
|||
} |
|||
|
|||
@Test |
|||
void testArgumentEntryType() { |
|||
assertThat(entry.getType()).isEqualTo(ArgumentEntryType.SINGLE_VALUE); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenRollingEntryPassed() { |
|||
assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) |
|||
.isInstanceOf(IllegalArgumentException.class) |
|||
.hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWithThaSameTs() { |
|||
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenNewVersionIsNull() { |
|||
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, new LongDataEntry("key", 13L), null))).isTrue(); |
|||
assertThat(entry.getValue()).isEqualTo(13L); |
|||
assertThat(entry.getVersion()).isNull(); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { |
|||
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 369L))).isTrue(); |
|||
assertThat(entry.getValue()).isEqualTo(18L); |
|||
assertThat(entry.getVersion()).isEqualTo(369L); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenNewVersionIsLessThanCurrent() { |
|||
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenValueWasNotChanged() { |
|||
assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); |
|||
} |
|||
} |
|||
@ -0,0 +1,123 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import org.junit.jupiter.api.BeforeEach; |
|||
import org.junit.jupiter.api.Test; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
|
|||
import java.util.Map; |
|||
import java.util.TreeMap; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.assertj.core.api.Assertions.assertThatThrownBy; |
|||
|
|||
public class TsRollingArgumentEntryTest { |
|||
|
|||
private TsRollingArgumentEntry entry; |
|||
|
|||
private final long ts = System.currentTimeMillis(); |
|||
|
|||
@BeforeEach |
|||
void setUp() { |
|||
TreeMap<Long, Double> values = new TreeMap<>(); |
|||
values.put(ts - 40, 10.0); |
|||
values.put(ts - 30, 12.0); |
|||
values.put(ts - 20, 17.0); |
|||
|
|||
entry = new TsRollingArgumentEntry(5, 30000L, values); |
|||
} |
|||
|
|||
@Test |
|||
void testArgumentEntryType() { |
|||
assertThat(entry.getType()).isEqualTo(ArgumentEntryType.TS_ROLLING); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenSingleValueEntryPassed() { |
|||
SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new DoubleDataEntry("key", 23.0), 123L); |
|||
|
|||
assertThat(entry.updateEntry(newEntry)).isTrue(); |
|||
assertThat(entry.getTsRecords()).hasSize(4); |
|||
assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenRollingEntryPassed() { |
|||
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); |
|||
TreeMap<Long, Double> values = new TreeMap<>(); |
|||
values.put(ts - 10, 7.0); |
|||
values.put(ts - 5, 1.0); |
|||
newEntry.setTsRecords(values); |
|||
|
|||
assertThat(entry.updateEntry(newEntry)).isTrue(); |
|||
assertThat(entry.getTsRecords()).hasSize(5); |
|||
assertThat(entry.getTsRecords()).isEqualTo(Map.of( |
|||
ts - 40, 10.0, |
|||
ts - 30, 12.0, |
|||
ts - 20, 17.0, |
|||
ts - 10, 7.0, |
|||
ts - 5, 1.0 |
|||
)); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenValueIsNotNumber() { |
|||
SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new StringDataEntry("key", "string"), 123L); |
|||
|
|||
assertThat(entry.updateEntry(newEntry)).isTrue(); |
|||
assertThat(entry.getTsRecords().get(ts - 10)).isNaN(); |
|||
} |
|||
|
|||
@Test |
|||
void testUpdateEntryWhenOldTelemetry() { |
|||
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); |
|||
TreeMap<Long, Double> values = new TreeMap<>(); |
|||
values.put(ts - 40000, 4.0);// will not be used for calculation
|
|||
values.put(ts - 45000, 2.0);// will not be used for calculation
|
|||
values.put(ts - 5, 0.0); |
|||
newEntry.setTsRecords(values); |
|||
|
|||
entry = new TsRollingArgumentEntry(3, 30000L); |
|||
assertThat(entry.updateEntry(newEntry)).isTrue(); |
|||
assertThat(entry.getTsRecords()).hasSize(1); |
|||
assertThat(entry.getTsRecords()).isEqualTo(Map.of( |
|||
ts - 5, 0.0 |
|||
)); |
|||
} |
|||
|
|||
@Test |
|||
void testPerformCalculationWhenArgumentsMoreThanLimit() { |
|||
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); |
|||
TreeMap<Long, Double> values = new TreeMap<>(); |
|||
values.put(ts - 20, 1000.0);// will not be used
|
|||
values.put(ts - 18, 0.0); |
|||
values.put(ts - 16, 0.0); |
|||
values.put(ts - 14, 0.0); |
|||
newEntry.setTsRecords(values); |
|||
|
|||
entry = new TsRollingArgumentEntry(3, 30000L); |
|||
assertThat(entry.updateEntry(newEntry)).isTrue(); |
|||
assertThat(entry.getTsRecords()).hasSize(3); |
|||
assertThat(entry.getTsRecords()).isEqualTo(Map.of( |
|||
ts - 18, 0.0, |
|||
ts - 16, 0.0, |
|||
ts - 14, 0.0 |
|||
)); |
|||
} |
|||
|
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue