trikimiki
1 year ago
795 changed files with 39537 additions and 2808 deletions
@ -0,0 +1,70 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.DebugModeUtil; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.service.ContextAwareActor; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.TbActorMsg; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
|
|||
@Slf4j |
|||
public abstract class AbstractCalculatedFieldActor extends ContextAwareActor { |
|||
|
|||
protected final TenantId tenantId; |
|||
|
|||
public AbstractCalculatedFieldActor(ActorSystemContext systemContext, TenantId tenantId) { |
|||
super(systemContext); |
|||
this.tenantId = tenantId; |
|||
} |
|||
|
|||
@Override |
|||
protected boolean doProcess(TbActorMsg msg) { |
|||
if (msg instanceof ToCalculatedFieldSystemMsg cfm) { |
|||
Exception cause; |
|||
try { |
|||
return doProcessCfMsg(cfm); |
|||
} catch (CalculatedFieldException cfe) { |
|||
if (DebugModeUtil.isDebugFailuresAvailable(cfe.getCtx().getCalculatedField())) { |
|||
String message; |
|||
if (cfe.getErrorMessage() != null) { |
|||
message = cfe.getErrorMessage(); |
|||
} else if (cfe.getCause() != null) { |
|||
message = cfe.getCause().getMessage(); |
|||
} else { |
|||
message = "N/A"; |
|||
} |
|||
systemContext.persistCalculatedFieldDebugEvent(tenantId, cfe.getCtx().getCfId(), cfe.getEventEntity(), cfe.getArguments(), cfe.getMsgId(), cfe.getMsgType(), null, message); |
|||
} |
|||
cause = cfe.getCause(); |
|||
} catch (Exception e) { |
|||
logProcessingException(e); |
|||
cause = e; |
|||
} |
|||
cfm.getCallback().onFailure(cause); |
|||
return true; |
|||
} else { |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
abstract void logProcessingException(Exception e); |
|||
|
|||
abstract boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException; |
|||
|
|||
} |
|||
@ -0,0 +1,81 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.TbActorException; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
|
|||
@Slf4j |
|||
public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { |
|||
|
|||
private final CalculatedFieldEntityMessageProcessor processor; |
|||
|
|||
CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { |
|||
super(systemContext, tenantId); |
|||
this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); |
|||
} |
|||
|
|||
@Override |
|||
public void init(TbActorCtx ctx) throws TbActorException { |
|||
super.init(ctx); |
|||
log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); |
|||
try { |
|||
processor.init(ctx); |
|||
log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); |
|||
} catch (Exception e) { |
|||
log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); |
|||
throw new TbActorException("Failed to initialize CF entity actor", e); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { |
|||
switch (msg.getMsgType()) { |
|||
case CF_PARTITIONS_CHANGE_MSG: |
|||
processor.process((CalculatedFieldPartitionChangeMsg) msg); |
|||
break; |
|||
case CF_STATE_RESTORE_MSG: |
|||
processor.process((CalculatedFieldStateRestoreMsg) msg); |
|||
break; |
|||
case CF_ENTITY_INIT_CF_MSG: |
|||
processor.process((EntityInitCalculatedFieldMsg) msg); |
|||
break; |
|||
case CF_ENTITY_DELETE_MSG: |
|||
processor.process((CalculatedFieldEntityDeleteMsg) msg); |
|||
break; |
|||
case CF_ENTITY_TELEMETRY_MSG: |
|||
processor.process((EntityCalculatedFieldTelemetryMsg) msg); |
|||
break; |
|||
case CF_LINKED_TELEMETRY_MSG: |
|||
processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); |
|||
break; |
|||
default: |
|||
return false; |
|||
} |
|||
return true; |
|||
} |
|||
|
|||
@Override |
|||
void logProcessingException(Exception e) { |
|||
log.warn("[{}][{}] Processing failure", tenantId, processor.entityId, e); |
|||
} |
|||
} |
|||
@ -0,0 +1,50 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActor; |
|||
import org.thingsboard.server.actors.TbActorId; |
|||
import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; |
|||
import org.thingsboard.server.actors.TbEntityActorId; |
|||
import org.thingsboard.server.actors.device.DeviceActor; |
|||
import org.thingsboard.server.actors.service.ContextBasedCreator; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
|
|||
public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { |
|||
super(context); |
|||
this.tenantId = tenantId; |
|||
this.entityId = entityId; |
|||
} |
|||
|
|||
@Override |
|||
public TbActorId createActorId() { |
|||
return new TbCalculatedFieldEntityActorId(entityId); |
|||
} |
|||
|
|||
@Override |
|||
public TbActor createActor() { |
|||
return new CalculatedFieldEntityActor(context, tenantId, entityId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,44 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
|
|||
@Data |
|||
public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final TbCallback callback; |
|||
|
|||
public CalculatedFieldEntityDeleteMsg(TenantId tenantId, |
|||
EntityId entityId, |
|||
TbCallback callback) { |
|||
this.tenantId = tenantId; |
|||
this.entityId = entityId; |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_ENTITY_DELETE_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,444 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.SneakyThrows; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.DebugModeUtil; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.StringUtils; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.LinkedList; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.stream.Collectors; |
|||
|
|||
|
|||
/** |
|||
* @author Andrew Shvayka |
|||
*/ |
|||
@Slf4j |
|||
public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { |
|||
// (1 for result persistence + 1 for the state persistence )
|
|||
public static final int CALLBACKS_PER_CF = 2; |
|||
|
|||
final TenantId tenantId; |
|||
final EntityId entityId; |
|||
final CalculatedFieldProcessingService cfService; |
|||
final CalculatedFieldStateService cfStateService; |
|||
final int partition; |
|||
|
|||
TbActorCtx ctx; |
|||
Map<CalculatedFieldId, CalculatedFieldState> states = new HashMap<>(); |
|||
|
|||
CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { |
|||
super(systemContext); |
|||
this.tenantId = tenantId; |
|||
this.entityId = entityId; |
|||
this.cfService = systemContext.getCalculatedFieldProcessingService(); |
|||
this.cfStateService = systemContext.getCalculatedFieldStateService(); |
|||
this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); |
|||
} |
|||
|
|||
void init(TbActorCtx ctx) { |
|||
this.ctx = ctx; |
|||
} |
|||
|
|||
public void process(CalculatedFieldPartitionChangeMsg msg) { |
|||
if (!msg.getPartitions()[partition]) { |
|||
log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); |
|||
ctx.stop(ctx.getSelf()); |
|||
} |
|||
} |
|||
|
|||
public void process(CalculatedFieldStateRestoreMsg msg) { |
|||
CalculatedFieldId cfId = msg.getId().cfId(); |
|||
log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); |
|||
if (msg.getState() != null) { |
|||
states.put(cfId, msg.getState()); |
|||
} else { |
|||
states.remove(cfId); |
|||
} |
|||
} |
|||
|
|||
public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); |
|||
var ctx = msg.getCtx(); |
|||
if (msg.isForceReinit()) { |
|||
log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); |
|||
states.remove(ctx.getCfId()); |
|||
} |
|||
try { |
|||
var state = getOrInitState(ctx); |
|||
if (state.isSizeOk()) { |
|||
processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); |
|||
} else { |
|||
throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); |
|||
} |
|||
} catch (Exception e) { |
|||
if (e instanceof CalculatedFieldException cfe) { |
|||
throw cfe; |
|||
} |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
|||
} |
|||
} |
|||
|
|||
public void process(CalculatedFieldEntityDeleteMsg msg) { |
|||
log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); |
|||
if (this.entityId.equals(msg.getEntityId())) { |
|||
if (states.isEmpty()) { |
|||
msg.getCallback().onSuccess(); |
|||
} else { |
|||
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); |
|||
states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); |
|||
ctx.stop(ctx.getSelf()); |
|||
} |
|||
} else { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
var state = states.remove(cfId); |
|||
if (state != null) { |
|||
cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); |
|||
} else { |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); |
|||
var proto = msg.getProto(); |
|||
var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); |
|||
MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); |
|||
List<CalculatedFieldId> cfIdList = getCalculatedFieldIds(proto); |
|||
Set<CalculatedFieldId> cfIdSet = new HashSet<>(cfIdList); |
|||
for (var ctx : msg.getEntityIdFields()) { |
|||
process(ctx, proto, cfIdSet, cfIdList, callback); |
|||
} |
|||
for (var ctx : msg.getProfileIdFields()) { |
|||
process(ctx, proto, cfIdSet, cfIdList, callback); |
|||
} |
|||
} |
|||
|
|||
public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); |
|||
var proto = msg.getProto(); |
|||
var ctx = msg.getCtx(); |
|||
var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); |
|||
try { |
|||
List<CalculatedFieldId> cfIds = getCalculatedFieldIds(proto); |
|||
if (cfIds.contains(ctx.getCfId())) { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} else { |
|||
if (proto.getTsDataCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else if (proto.getAttrDataCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else if (proto.getRemovedTsKeysCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else if (proto.getRemovedAttrKeysCount() > 0) { |
|||
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} else { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
|||
} |
|||
} |
|||
|
|||
private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection<CalculatedFieldId> cfIds, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
try { |
|||
if (cfIds.contains(ctx.getCfId())) { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} else { |
|||
if (proto.getTsDataCount() > 0) { |
|||
processTelemetry(ctx, proto, cfIdList, callback); |
|||
} else if (proto.getAttrDataCount() > 0) { |
|||
processAttributes(ctx, proto, cfIdList, callback); |
|||
} else if (proto.getRemovedTsKeysCount() > 0) { |
|||
processRemovedTelemetry(ctx, proto, cfIdList, callback); |
|||
} else if (proto.getRemovedAttrKeysCount() > 0) { |
|||
processRemovedAttributes(ctx, proto, cfIdList, callback); |
|||
} else { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
if (e instanceof CalculatedFieldException cfe) { |
|||
throw cfe; |
|||
} |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
|||
} |
|||
} |
|||
|
|||
private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
|||
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithDefaultValue(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
|||
} |
|||
|
|||
private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback, |
|||
Map<String, ArgumentEntry> newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { |
|||
if (newArgValues.isEmpty()) { |
|||
log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
CalculatedFieldState state = states.get(ctx.getCfId()); |
|||
boolean justRestored = false; |
|||
if (state == null) { |
|||
state = getOrInitState(ctx); |
|||
justRestored = true; |
|||
} |
|||
if (state.isSizeOk()) { |
|||
if (state.updateState(ctx, newArgValues) || justRestored) { |
|||
cfIdList = new ArrayList<>(cfIdList); |
|||
cfIdList.add(ctx.getCfId()); |
|||
processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); |
|||
} else { |
|||
callback.onSuccess(CALLBACKS_PER_CF); |
|||
} |
|||
} else { |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); |
|||
} |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { |
|||
CalculatedFieldState state = states.get(ctx.getCfId()); |
|||
if (state != null) { |
|||
return state; |
|||
} else { |
|||
ListenableFuture<CalculatedFieldState> stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); |
|||
// Ugly but necessary. We do not expect to often fetch data from DB. Only once per <Entity, CalculatedField> pair lifetime.
|
|||
// This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low.
|
|||
// Alternatively, we can fetch the state outside the actor system and push separate command to create this actor,
|
|||
// but this will significantly complicate the code.
|
|||
state = stateFuture.get(1, TimeUnit.MINUTES); |
|||
state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); |
|||
states.put(ctx.getCfId(), state); |
|||
} |
|||
return state; |
|||
} |
|||
|
|||
private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { |
|||
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); |
|||
boolean stateSizeChecked = false; |
|||
try { |
|||
if (ctx.isInitialized() && state.isReady()) { |
|||
CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); |
|||
state.checkStateSize(ctxId, ctx.getMaxStateSize()); |
|||
stateSizeChecked = true; |
|||
if (state.isSizeOk()) { |
|||
cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); |
|||
if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { |
|||
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); |
|||
} |
|||
} |
|||
} |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); |
|||
} finally { |
|||
if (!stateSizeChecked) { |
|||
state.checkStateSize(ctxId, ctx.getMaxStateSize()); |
|||
} |
|||
if (state.isSizeOk()) { |
|||
cfStateService.persistState(ctxId, state, callback); |
|||
} else { |
|||
removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException { |
|||
// We remove the state, but remember that it is over-sized in a local map.
|
|||
cfStateService.removeState(ctxId, new TbCallback() { |
|||
@Override |
|||
public void onSuccess() { |
|||
callback.onFailure(ex); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(ex); |
|||
} |
|||
}); |
|||
throw ex; |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) { |
|||
return mapToArguments(ctx.getMainEntityArguments(), data); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List<TsKvProto> data) { |
|||
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
return mapToArguments(argNames, data); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, List<TsKvProto> data) { |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
|||
for (TsKvProto item : data) { |
|||
ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); |
|||
String argName = argNames.get(key); |
|||
if (argName != null) { |
|||
arguments.put(argName, new SingleValueArgumentEntry(item)); |
|||
} |
|||
key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); |
|||
argName = argNames.get(key); |
|||
if (argName != null) { |
|||
arguments.put(argName, new SingleValueArgumentEntry(item)); |
|||
} |
|||
} |
|||
return arguments; |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
|||
return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
|||
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
return mapToArguments(argNames, scope, attrDataList); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
|||
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
|||
for (AttributeValueProto item : attrDataList) { |
|||
ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); |
|||
String argName = argNames.get(key); |
|||
if (argName != null) { |
|||
arguments.put(argName, new SingleValueArgumentEntry(item)); |
|||
} |
|||
} |
|||
return arguments; |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List<String> removedAttrKeys) { |
|||
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
|||
if (argNames.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List<String> removedAttrKeys) { |
|||
return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(Map<ReferencedEntityKey, String> argNames, Map<String, Argument> configArguments, AttributeScopeProto scope, List<String> removedAttrKeys) { |
|||
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
|||
for (String removedKey : removedAttrKeys) { |
|||
ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); |
|||
String argName = argNames.get(key); |
|||
if (argName != null) { |
|||
Argument argument = configArguments.get(argName); |
|||
String defaultValue = (argument != null) ? argument.getDefaultValue() : null; |
|||
arguments.put(argName, StringUtils.isNotEmpty(defaultValue) |
|||
? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) |
|||
: new SingleValueArgumentEntry()); |
|||
|
|||
} |
|||
} |
|||
return arguments; |
|||
} |
|||
|
|||
private Map<String, ArgumentEntry> mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List<String> removedTelemetryKeys) { |
|||
Map<String, Argument> deletedArguments = ctx.getArguments().entrySet().stream() |
|||
.filter(entry -> removedTelemetryKeys.contains(entry.getKey())) |
|||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); |
|||
|
|||
Map<String, ArgumentEntry> fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); |
|||
|
|||
fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); |
|||
return fetchedArgs; |
|||
} |
|||
|
|||
private static List<CalculatedFieldId> getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { |
|||
List<CalculatedFieldId> cfIds = new LinkedList<>(); |
|||
for (var cfId : proto.getPreviousCalculatedFieldsList()) { |
|||
cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); |
|||
} |
|||
return cfIds; |
|||
} |
|||
|
|||
private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { |
|||
if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { |
|||
return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); |
|||
} |
|||
return null; |
|||
} |
|||
|
|||
private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { |
|||
if (!proto.getTbMsgType().isEmpty()) { |
|||
return TbMsgType.valueOf(proto.getTbMsgType()); |
|||
} |
|||
return null; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Builder; |
|||
import lombok.Getter; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
|
|||
@Getter |
|||
@Builder |
|||
public class CalculatedFieldException extends Exception { |
|||
|
|||
private final CalculatedFieldCtx ctx; |
|||
private final EntityId eventEntity; |
|||
private final UUID msgId; |
|||
private final TbMsgType msgType; |
|||
private Map<String, ArgumentEntry> arguments; |
|||
private String errorMessage; |
|||
private Exception cause; |
|||
|
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
|
|||
@Data |
|||
public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldLinkedTelemetryMsgProto proto; |
|||
private final TbCallback callback; |
|||
|
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_LINKED_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,90 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.TbActorException; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
|
|||
/** |
|||
* Created by ashvayka on 15.03.18. |
|||
*/ |
|||
@Slf4j |
|||
public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { |
|||
|
|||
private final CalculatedFieldManagerMessageProcessor processor; |
|||
|
|||
public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { |
|||
super(systemContext, tenantId); |
|||
this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); |
|||
} |
|||
|
|||
@Override |
|||
public void init(TbActorCtx ctx) throws TbActorException { |
|||
super.init(ctx); |
|||
log.debug("[{}] Starting CF manager actor.", processor.tenantId); |
|||
try { |
|||
processor.init(ctx); |
|||
log.debug("[{}] CF manager actor started.", processor.tenantId); |
|||
} catch (Exception e) { |
|||
log.warn("[{}] Unknown failure", processor.tenantId, e); |
|||
throw new TbActorException("Failed to initialize manager actor", e); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { |
|||
switch (msg.getMsgType()) { |
|||
case CF_PARTITIONS_CHANGE_MSG: |
|||
processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); |
|||
break; |
|||
case CF_INIT_MSG: |
|||
processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); |
|||
break; |
|||
case CF_LINK_INIT_MSG: |
|||
processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); |
|||
break; |
|||
case CF_STATE_RESTORE_MSG: |
|||
processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); |
|||
break; |
|||
case CF_ENTITY_LIFECYCLE_MSG: |
|||
processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); |
|||
break; |
|||
case CF_TELEMETRY_MSG: |
|||
processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); |
|||
break; |
|||
case CF_LINKED_TELEMETRY_MSG: |
|||
processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); |
|||
break; |
|||
default: |
|||
return false; |
|||
} |
|||
return true; |
|||
} |
|||
|
|||
@Override |
|||
void logProcessingException(Exception e) { |
|||
log.warn("[{}] Processing failure", tenantId, e); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,46 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActor; |
|||
import org.thingsboard.server.actors.TbActorId; |
|||
import org.thingsboard.server.actors.TbEntityActorId; |
|||
import org.thingsboard.server.actors.TbStringActorId; |
|||
import org.thingsboard.server.actors.service.ContextBasedCreator; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { |
|||
|
|||
private final TenantId tenantId; |
|||
|
|||
public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { |
|||
super(context); |
|||
this.tenantId = tenantId; |
|||
} |
|||
|
|||
@Override |
|||
public TbActorId createActorId() { |
|||
return new TbStringActorId("CFM|" + tenantId); |
|||
} |
|||
|
|||
@Override |
|||
public TbActor createActor() { |
|||
return new CalculatedFieldManagerActor(context, tenantId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,468 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.TbActorCtx; |
|||
import org.thingsboard.server.actors.TbActorRef; |
|||
import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; |
|||
import org.thingsboard.server.actors.service.DefaultActorService; |
|||
import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.id.AssetId; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
|||
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.concurrent.CopyOnWriteArrayList; |
|||
|
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; |
|||
|
|||
|
|||
/** |
|||
* @author Andrew Shvayka |
|||
*/ |
|||
@Slf4j |
|||
public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { |
|||
|
|||
private final Map<CalculatedFieldId, CalculatedFieldCtx> calculatedFields = new HashMap<>(); |
|||
private final Map<EntityId, List<CalculatedFieldCtx>> entityIdCalculatedFields = new HashMap<>(); |
|||
private final Map<EntityId, List<CalculatedFieldLink>> entityIdCalculatedFieldLinks = new HashMap<>(); |
|||
|
|||
private final CalculatedFieldProcessingService cfExecService; |
|||
private final CalculatedFieldStateService cfStateService; |
|||
private final CalculatedFieldEntityProfileCache cfEntityCache; |
|||
private final CalculatedFieldService cfDaoService; |
|||
private final TbAssetProfileCache assetProfileCache; |
|||
private final TbDeviceProfileCache deviceProfileCache; |
|||
protected final TenantId tenantId; |
|||
|
|||
protected TbActorCtx ctx; |
|||
|
|||
CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { |
|||
super(systemContext); |
|||
this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); |
|||
this.cfExecService = systemContext.getCalculatedFieldProcessingService(); |
|||
this.cfStateService = systemContext.getCalculatedFieldStateService(); |
|||
this.cfDaoService = systemContext.getCalculatedFieldService(); |
|||
this.assetProfileCache = systemContext.getAssetProfileCache(); |
|||
this.deviceProfileCache = systemContext.getDeviceProfileCache(); |
|||
this.tenantId = tenantId; |
|||
} |
|||
|
|||
void init(TbActorCtx ctx) { |
|||
this.ctx = ctx; |
|||
} |
|||
|
|||
public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException { |
|||
log.info("[{}] Processing CF init message.", msg.getCf().getId()); |
|||
var cf = msg.getCf(); |
|||
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
|||
try { |
|||
cfCtx.init(); |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
|||
} |
|||
calculatedFields.put(cf.getId(), cfCtx); |
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
|
|||
public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { |
|||
log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); |
|||
var link = msg.getLink(); |
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
|
|||
public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { |
|||
var cfId = msg.getId().cfId(); |
|||
var calculatedField = calculatedFields.get(cfId); |
|||
|
|||
if (calculatedField != null) { |
|||
msg.getState().setRequiredArguments(calculatedField.getArgNames()); |
|||
log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); |
|||
getOrCreateActor(msg.getId().entityId()).tell(msg); |
|||
} else { |
|||
cfStateService.removeState(msg.getId(), msg.getCallback()); |
|||
} |
|||
} |
|||
|
|||
public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { |
|||
log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); |
|||
var entityType = msg.getData().getEntityId().getEntityType(); |
|||
var event = msg.getData().getEvent(); |
|||
switch (entityType) { |
|||
case CALCULATED_FIELD: { |
|||
switch (event) { |
|||
case CREATED: |
|||
onCfCreated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case UPDATED: |
|||
onCfUpdated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case DELETED: |
|||
onCfDeleted(msg.getData(), msg.getCallback()); |
|||
break; |
|||
default: |
|||
msg.getCallback().onSuccess(); |
|||
break; |
|||
} |
|||
break; |
|||
} |
|||
case DEVICE: |
|||
case ASSET: { |
|||
switch (event) { |
|||
case CREATED: |
|||
onEntityCreated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case UPDATED: |
|||
onEntityUpdated(msg.getData(), msg.getCallback()); |
|||
break; |
|||
case DELETED: |
|||
onEntityDeleted(msg.getData(), msg.getCallback()); |
|||
break; |
|||
default: |
|||
msg.getCallback().onSuccess(); |
|||
break; |
|||
} |
|||
break; |
|||
} |
|||
default: { |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
EntityId entityId = msg.getEntityId(); |
|||
EntityId profileId = getProfileId(tenantId, entityId); |
|||
cfEntityCache.add(tenantId, profileId, entityId); |
|||
var entityIdFields = getCalculatedFieldsByEntityId(entityId); |
|||
var profileIdFields = getCalculatedFieldsByEntityId(profileId); |
|||
var fieldsCount = entityIdFields.size() + profileIdFields.size(); |
|||
if (fieldsCount > 0) { |
|||
MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); |
|||
entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
|||
profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { |
|||
cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); |
|||
var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); |
|||
var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); |
|||
var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); |
|||
if (fieldsCount > 0) { |
|||
MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); |
|||
var entityId = msg.getEntityId(); |
|||
oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); |
|||
newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
cfEntityCache.evict(tenantId, msg.getEntityId()); |
|||
log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); |
|||
getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); |
|||
} |
|||
|
|||
private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
if (calculatedFields.containsKey(cfId)) { |
|||
log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
var cf = cfDaoService.findById(msg.getTenantId(), cfId); |
|||
if (cf == null) { |
|||
log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
|||
try { |
|||
cfCtx.init(); |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
|||
} |
|||
calculatedFields.put(cf.getId(), cfCtx); |
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); |
|||
addLinks(cf); |
|||
initCf(cfCtx, callback, false); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
var oldCfCtx = calculatedFields.get(cfId); |
|||
if (oldCfCtx == null) { |
|||
onCfCreated(msg, callback); |
|||
} else { |
|||
var newCf = cfDaoService.findById(msg.getTenantId(), cfId); |
|||
if (newCf == null) { |
|||
log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
|||
try { |
|||
newCfCtx.init(); |
|||
} catch (Exception e) { |
|||
throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
|||
} |
|||
calculatedFields.put(newCf.getId(), newCfCtx); |
|||
List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); |
|||
List<CalculatedFieldCtx> newCfList = new CopyOnWriteArrayList<>(); |
|||
boolean found = false; |
|||
for (CalculatedFieldCtx oldCtx : oldCfList) { |
|||
if (oldCtx.getCfId().equals(newCf.getId())) { |
|||
newCfList.add(newCfCtx); |
|||
found = true; |
|||
} else { |
|||
newCfList.add(oldCtx); |
|||
} |
|||
} |
|||
if (!found) { |
|||
newCfList.add(newCfCtx); |
|||
} |
|||
entityIdCalculatedFields.put(newCf.getEntityId(), newCfList); |
|||
|
|||
deleteLinks(oldCfCtx); |
|||
addLinks(newCf); |
|||
|
|||
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
|||
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
|||
var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); |
|||
if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { |
|||
initCf(newCfCtx, callback, stateChanges); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { |
|||
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
|||
var cfCtx = calculatedFields.remove(cfId); |
|||
if (cfCtx == null) { |
|||
log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); |
|||
callback.onSuccess(); |
|||
} else { |
|||
entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); |
|||
deleteLinks(cfCtx); |
|||
|
|||
EntityId entityId = cfCtx.getEntityId(); |
|||
EntityType entityType = cfCtx.getEntityId().getEntityType(); |
|||
if (isProfileEntity(entityType)) { |
|||
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); |
|||
if (!entityIds.isEmpty()) { |
|||
//TODO: no need to do this if we cache all created actors and know which one belong to us;
|
|||
var multiCallback = new MultipleTbCallback(entityIds.size(), callback); |
|||
entityIds.forEach(id -> deleteCfForEntity(id, cfId, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} else { |
|||
deleteCfForEntity(entityId, cfId, callback); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { |
|||
EntityId entityId = msg.getEntityId(); |
|||
log.info("Received telemetry msg from entity [{}]", entityId); |
|||
// 2 = 1 for CF processing + 1 for links processing
|
|||
MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); |
|||
// process all cfs related to entity, or it's profile;
|
|||
var entityIdFields = getCalculatedFieldsByEntityId(entityId); |
|||
var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); |
|||
if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { |
|||
log.info("Pushing telemetry msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
// process all links (if any);
|
|||
List<CalculatedFieldEntityCtxId> linkedCalculatedFields = filterCalculatedFieldLinks(msg); |
|||
var linksSize = linkedCalculatedFields.size(); |
|||
if (linksSize > 0) { |
|||
cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { |
|||
EntityId sourceEntityId = msg.getEntityId(); |
|||
log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); |
|||
var proto = msg.getProto(); |
|||
var linksList = proto.getLinksList(); |
|||
for (var linkProto : linksList) { |
|||
var link = fromProto(linkProto); |
|||
var targetEntityId = link.entityId(); |
|||
var targetEntityType = targetEntityId.getEntityType(); |
|||
var cf = calculatedFields.get(link.cfId()); |
|||
if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { |
|||
// iterate over all entities that belong to profile and push the message for corresponding CF
|
|||
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); |
|||
if (!entityIds.isEmpty()) { |
|||
MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); |
|||
var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); |
|||
entityIds.forEach(entityId -> { |
|||
log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(newMsg); |
|||
}); |
|||
} else { |
|||
msg.getCallback().onSuccess(); |
|||
} |
|||
} else { |
|||
log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); |
|||
var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); |
|||
getOrCreateActor(targetEntityId).tell(newMsg); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private List<CalculatedFieldEntityCtxId> filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { |
|||
EntityId entityId = msg.getEntityId(); |
|||
var proto = msg.getProto(); |
|||
List<CalculatedFieldEntityCtxId> result = new ArrayList<>(); |
|||
for (var link : getCalculatedFieldLinksByEntityId(entityId)) { |
|||
CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); |
|||
if (ctx.linkMatches(entityId, proto)) { |
|||
result.add(ctx.toCalculatedFieldEntityCtxId()); |
|||
} |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private List<CalculatedFieldCtx> getCalculatedFieldsByEntityId(EntityId entityId) { |
|||
if (entityId == null) { |
|||
return Collections.emptyList(); |
|||
} |
|||
var result = entityIdCalculatedFields.get(entityId); |
|||
if (result == null) { |
|||
result = Collections.emptyList(); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId) { |
|||
if (entityId == null) { |
|||
return Collections.emptyList(); |
|||
} |
|||
var result = entityIdCalculatedFieldLinks.get(entityId); |
|||
if (result == null) { |
|||
result = Collections.emptyList(); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { |
|||
EntityId entityId = cfCtx.getEntityId(); |
|||
EntityType entityType = cfCtx.getEntityId().getEntityType(); |
|||
if (isProfileEntity(entityType)) { |
|||
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); |
|||
if (!entityIds.isEmpty()) { |
|||
var multiCallback = new MultipleTbCallback(entityIds.size(), callback); |
|||
entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); |
|||
} else { |
|||
callback.onSuccess(); |
|||
} |
|||
} else { |
|||
initCfForEntity(entityId, cfCtx, forceStateReinit, callback); |
|||
} |
|||
} |
|||
|
|||
private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { |
|||
log.info("Pushing delete CF msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); |
|||
} |
|||
|
|||
private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { |
|||
log.info("Pushing entity init CF msg to specific actor [{}]", entityId); |
|||
getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); |
|||
} |
|||
|
|||
private static boolean isProfileEntity(EntityType entityType) { |
|||
return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); |
|||
} |
|||
|
|||
private EntityId getProfileId(TenantId tenantId, EntityId entityId) { |
|||
return switch (entityId.getEntityType()) { |
|||
case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); |
|||
case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); |
|||
default -> null; |
|||
}; |
|||
} |
|||
|
|||
private TbActorRef getOrCreateActor(EntityId entityId) { |
|||
return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), |
|||
() -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, |
|||
() -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), |
|||
() -> true); |
|||
} |
|||
|
|||
private void addLinks(CalculatedField newCf) { |
|||
var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); |
|||
newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link)); |
|||
} |
|||
|
|||
private void deleteLinks(CalculatedFieldCtx cfCtx) { |
|||
var oldCf = cfCtx.getCalculatedField(); |
|||
var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); |
|||
oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).remove(link)); |
|||
} |
|||
|
|||
public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { |
|||
ctx.broadcastToChildren(msg, true); |
|||
} |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
@Data |
|||
public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final CalculatedFieldEntityCtxId id; |
|||
private final CalculatedFieldState state; |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_STATE_RESTORE_MSG; |
|||
} |
|||
|
|||
@Override |
|||
public TenantId getTenantId() { |
|||
return id.tenantId(); |
|||
} |
|||
} |
|||
@ -0,0 +1,39 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
|
|||
@Data |
|||
public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldTelemetryMsgProto proto; |
|||
private final TbCallback callback; |
|||
|
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,42 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Data |
|||
public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldTelemetryMsgProto proto; |
|||
private final CalculatedFieldCtx ctx; |
|||
private final TbCallback callback; |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_LINKED_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Data |
|||
public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final EntityId entityId; |
|||
private final CalculatedFieldTelemetryMsgProto proto; |
|||
// Both lists are effectively immutable in CalculatedFieldManagerMessageProcessor and must stay so.
|
|||
private final List<CalculatedFieldCtx> entityIdFields; |
|||
private final List<CalculatedFieldCtx> profileIdFields; |
|||
private final TbCallback callback; |
|||
|
|||
public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, |
|||
List<CalculatedFieldCtx> entityIdFields, |
|||
List<CalculatedFieldCtx> profileIdFields, |
|||
TbCallback callback) { |
|||
this.tenantId = msg.getTenantId(); |
|||
this.entityId = msg.getEntityId(); |
|||
this.proto = msg.getProto(); |
|||
this.entityIdFields = entityIdFields; |
|||
this.profileIdFields = profileIdFields; |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_ENTITY_TELEMETRY_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,41 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.MsgType; |
|||
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Data |
|||
public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { |
|||
|
|||
private final TenantId tenantId; |
|||
private final CalculatedFieldCtx ctx; |
|||
private final TbCallback callback; |
|||
private final boolean forceReinit; |
|||
|
|||
@Override |
|||
public MsgType getMsgType() { |
|||
return MsgType.CF_ENTITY_INIT_CF_MSG; |
|||
} |
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.actors.calculatedField; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
|
|||
import java.util.UUID; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
@Slf4j |
|||
public class MultipleTbCallback implements TbCallback { |
|||
@Getter |
|||
private final UUID id; |
|||
private final AtomicInteger counter; |
|||
private final TbCallback callback; |
|||
|
|||
public MultipleTbCallback(int count, TbCallback callback) { |
|||
id = UUID.randomUUID(); |
|||
this.counter = new AtomicInteger(count); |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public void onSuccess() { |
|||
onSuccess(1); |
|||
} |
|||
|
|||
public void onSuccess(int number) { |
|||
log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); |
|||
if (counter.addAndGet(-number) <= 0) { |
|||
log.trace("[{}][{}] Done.", id, callback.getId()); |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
log.warn("[{}][{}] onFailure.", id, callback.getId()); |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
@ -0,0 +1,283 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.controller; |
|||
|
|||
import com.fasterxml.jackson.core.type.TypeReference; |
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.fasterxml.jackson.databind.node.ObjectNode; |
|||
import io.swagger.v3.oas.annotations.Parameter; |
|||
import io.swagger.v3.oas.annotations.media.Schema; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.http.HttpStatus; |
|||
import org.springframework.security.access.prepost.PreAuthorize; |
|||
import org.springframework.web.bind.annotation.PathVariable; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RequestMethod; |
|||
import org.springframework.web.bind.annotation.RequestParam; |
|||
import org.springframework.web.bind.annotation.ResponseBody; |
|||
import org.springframework.web.bind.annotation.ResponseStatus; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.script.api.tbel.TbelCfCtx; |
|||
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.EventInfo; |
|||
import org.thingsboard.server.common.data.HasTenantId; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.event.EventType; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.HasId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.config.annotations.ApiOperation; |
|||
import org.thingsboard.server.dao.event.EventService; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; |
|||
import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
import org.thingsboard.server.service.security.permission.Operation; |
|||
import org.thingsboard.server.service.security.permission.Resource; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Objects; |
|||
import java.util.Optional; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; |
|||
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; |
|||
import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; |
|||
import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; |
|||
import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; |
|||
import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; |
|||
|
|||
@RestController |
|||
@TbCoreComponent |
|||
@RequestMapping("/api") |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class CalculatedFieldController extends BaseController { |
|||
|
|||
private final TbCalculatedFieldService tbCalculatedFieldService; |
|||
private final EventService eventService; |
|||
private final TbelInvokeService tbelInvokeService; |
|||
|
|||
public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; |
|||
|
|||
public static final int TIMEOUT = 20; |
|||
|
|||
private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" |
|||
+ MARKDOWN_CODE_BLOCK_START |
|||
+ "{\n" + |
|||
" \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" + |
|||
" \"arguments\": {\n" + |
|||
" \"temperature\": {\n" + |
|||
" \"type\": \"TS_ROLLING\",\n" + |
|||
" \"timeWindow\": {\n" + |
|||
" \"startTs\": 1739775630002,\n" + |
|||
" \"endTs\": 65432211,\n" + |
|||
" \"limit\": 5\n" + |
|||
" },\n" + |
|||
" \"values\": [\n" + |
|||
" { \"ts\": 1739775639851, \"value\": 23 },\n" + |
|||
" { \"ts\": 1739775664561, \"value\": 43 },\n" + |
|||
" { \"ts\": 1739775713079, \"value\": 15 },\n" + |
|||
" { \"ts\": 1739775999522, \"value\": 34 },\n" + |
|||
" { \"ts\": 1739776228452, \"value\": 22 }\n" + |
|||
" ]\n" + |
|||
" },\n" + |
|||
" \"humidity\": { \"type\": \"SINGLE_VALUE\", \"ts\": 1739776478057, \"value\": 23 }\n" + |
|||
" }\n" + |
|||
"}" |
|||
+ MARKDOWN_CODE_BLOCK_END |
|||
+ "\n\n Expected result JSON contains \"output\" and \"error\"."; |
|||
|
|||
@ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", |
|||
notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + |
|||
"The newly created Calculated Field Id will be present in the response. " + |
|||
"Specify existing Calculated Field Id to update the calculated field. " + |
|||
"Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + |
|||
"Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " |
|||
+ TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") |
|||
@RequestBody CalculatedField calculatedField) throws Exception { |
|||
calculatedField.setTenantId(getTenantId()); |
|||
checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); |
|||
checkReferencedEntities(calculatedField.getConfiguration(), getCurrentUser()); |
|||
return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); |
|||
} |
|||
|
|||
@ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", |
|||
notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." |
|||
) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { |
|||
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
|||
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
|||
CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); |
|||
checkNotNull(calculatedField); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); |
|||
return calculatedField; |
|||
} |
|||
|
|||
@ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", |
|||
notes = "Fetch the Calculated Fields based on the provided Entity Id." |
|||
) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public PageData<CalculatedField> getCalculatedFieldsByEntityId( |
|||
@Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, |
|||
@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, |
|||
@Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, |
|||
@Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, |
|||
@Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, |
|||
@Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, |
|||
@Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { |
|||
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); |
|||
checkParameter("entityId", entityIdStr); |
|||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); |
|||
checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); |
|||
return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); |
|||
} |
|||
|
|||
@ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", |
|||
notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) |
|||
@ResponseStatus(value = HttpStatus.OK) |
|||
public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws Exception { |
|||
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
|||
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
|||
CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); |
|||
tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); |
|||
} |
|||
|
|||
@ApiOperation(value = "Get latest calculated field debug event (getLatestCalculatedFieldDebugEvent)", |
|||
notes = "Gets latest calculated field debug event for specified calculated field id. " + |
|||
"Referencing non-existing calculated field id will cause an error. " + TENANT_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/{calculatedFieldId}/debug", method = RequestMethod.GET) |
|||
@ResponseBody |
|||
public JsonNode getLatestCalculatedFieldDebugEvent(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { |
|||
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
|||
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
|||
CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.READ); |
|||
checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); |
|||
TenantId tenantId = getCurrentUser().getTenantId(); |
|||
return Optional.ofNullable(eventService.findLatestEvents(tenantId, calculatedFieldId, EventType.DEBUG_CALCULATED_FIELD, 1)) |
|||
.flatMap(events -> events.stream().map(EventInfo::getBody).findFirst()) |
|||
.orElse(null); |
|||
} |
|||
|
|||
@ApiOperation(value = "Test Script expression", |
|||
notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) |
|||
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
|||
@RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) |
|||
@ResponseBody |
|||
public JsonNode testScript( |
|||
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") |
|||
@RequestBody JsonNode inputParams) { |
|||
String expression = inputParams.get("expression").asText(); |
|||
Map<String, TbelCfArg> arguments = Objects.requireNonNullElse( |
|||
JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference<>() { |
|||
}), |
|||
Collections.emptyMap() |
|||
); |
|||
|
|||
ArrayList<String> ctxAndArgNames = new ArrayList<>(arguments.size() + 1); |
|||
ctxAndArgNames.add("ctx"); |
|||
ctxAndArgNames.addAll(arguments.keySet()); |
|||
|
|||
String output = ""; |
|||
String errorText = ""; |
|||
|
|||
try { |
|||
if (tbelInvokeService == null) { |
|||
throw new IllegalArgumentException("TBEL script engine is disabled!"); |
|||
} |
|||
|
|||
CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( |
|||
getTenantId(), |
|||
tbelInvokeService, |
|||
expression, |
|||
ctxAndArgNames.toArray(String[]::new) |
|||
); |
|||
|
|||
|
|||
Object[] args = new Object[ctxAndArgNames.size()]; |
|||
args[0] = new TbelCfCtx(arguments); |
|||
for (int i = 1; i < ctxAndArgNames.size(); i++) { |
|||
var arg = arguments.get(ctxAndArgNames.get(i)); |
|||
if (arg instanceof TbelCfSingleValueArg svArg) { |
|||
args[i] = svArg.getValue(); |
|||
} else { |
|||
args[i] = arg; |
|||
} |
|||
} |
|||
|
|||
JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); |
|||
output = JacksonUtil.toString(json); |
|||
} catch (Exception e) { |
|||
log.error("Error evaluating expression", e); |
|||
errorText = e.getMessage(); |
|||
} |
|||
|
|||
ObjectNode result = JacksonUtil.newObjectNode(); |
|||
result.put("output", output); |
|||
result.put("error", errorText); |
|||
return result; |
|||
} |
|||
|
|||
private <E extends HasId<I> & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { |
|||
List<EntityId> referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); |
|||
for (EntityId referencedEntityId : referencedEntityIds) { |
|||
EntityType entityType = referencedEntityId.getEntityType(); |
|||
switch (entityType) { |
|||
case TENANT, CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); |
|||
default -> |
|||
throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); |
|||
} |
|||
} |
|||
|
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.exception; |
|||
|
|||
public class CalculatedFieldStateException extends RuntimeException { |
|||
|
|||
public CalculatedFieldStateException(String message) { |
|||
super(message); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,72 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.exception.CalculatedFieldStateException; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; |
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
|||
|
|||
public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { |
|||
|
|||
@Autowired |
|||
private ActorSystemContext actorSystemContext; |
|||
|
|||
protected PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer; |
|||
|
|||
@Override |
|||
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer) { |
|||
this.eventConsumer = eventConsumer; |
|||
} |
|||
|
|||
@Override |
|||
public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { |
|||
if (state.isSizeExceedsLimit()) { |
|||
throw new CalculatedFieldStateException("State size exceeds the maximum allowed limit. The state will not be persisted to RocksDB."); |
|||
} |
|||
doPersist(stateId, toProto(stateId, state), callback); |
|||
} |
|||
|
|||
protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback); |
|||
|
|||
@Override |
|||
public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
|||
doRemove(stateId, callback); |
|||
} |
|||
|
|||
protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); |
|||
|
|||
protected void processRestoredState(CalculatedFieldStateProto stateMsg) { |
|||
var id = fromProto(stateMsg.getId()); |
|||
var state = fromProto(stateMsg); |
|||
processRestoredState(id, state); |
|||
} |
|||
|
|||
protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { |
|||
actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state)); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,45 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.List; |
|||
|
|||
public interface CalculatedFieldCache { |
|||
|
|||
CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); |
|||
|
|||
List<CalculatedField> getCalculatedFieldsByEntityId(EntityId entityId); |
|||
|
|||
List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId); |
|||
|
|||
CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); |
|||
|
|||
List<CalculatedFieldCtx> getCalculatedFieldCtxsByEntityId(EntityId entityId); |
|||
|
|||
void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); |
|||
|
|||
void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); |
|||
|
|||
void evict(CalculatedFieldId calculatedFieldId); |
|||
|
|||
} |
|||
@ -0,0 +1,19 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
public interface CalculatedFieldInitService { |
|||
} |
|||
@ -0,0 +1,43 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
public interface CalculatedFieldProcessingService { |
|||
|
|||
ListenableFuture<CalculatedFieldState> fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); |
|||
|
|||
Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments); |
|||
|
|||
void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List<CalculatedFieldId> cfIds, TbCallback callback); |
|||
|
|||
void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback); |
|||
|
|||
} |
|||
@ -0,0 +1,44 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import org.thingsboard.rule.engine.api.AttributesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.AttributesSaveRequest; |
|||
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; |
|||
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; |
|||
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; |
|||
|
|||
import java.util.List; |
|||
|
|||
public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQueueService { |
|||
|
|||
/** |
|||
* Filter CFs based on the request entity. Push to the queue if any matching CF exist; |
|||
* |
|||
* @param request - telemetry save request; |
|||
* @param callback |
|||
*/ |
|||
void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback<Void> callback); |
|||
|
|||
void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback); |
|||
|
|||
void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback); |
|||
|
|||
void pushRequestToQueue(TimeseriesDeleteRequest request, List<String> result, FutureCallback<Void> callback); |
|||
|
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
|
|||
@Data |
|||
public final class CalculatedFieldResult { |
|||
|
|||
private final OutputType type; |
|||
private final AttributeScope scope; |
|||
private final JsonNode result; |
|||
|
|||
} |
|||
@ -0,0 +1,41 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.exception.CalculatedFieldStateException; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
import java.util.Set; |
|||
|
|||
public interface CalculatedFieldStateService { |
|||
|
|||
void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer); |
|||
|
|||
void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; |
|||
|
|||
void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); |
|||
|
|||
void restore(Set<TopicPartitionInfo> partitions); |
|||
|
|||
void stop(); |
|||
|
|||
} |
|||
@ -0,0 +1,47 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import org.rocksdb.Options; |
|||
import org.rocksdb.WriteOptions; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.server.edqs.util.TbRocksDb; |
|||
|
|||
@Component |
|||
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") |
|||
public class CfRocksDb extends TbRocksDb { |
|||
|
|||
public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) { |
|||
super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); |
|||
} |
|||
|
|||
@PostConstruct |
|||
@Override |
|||
public void init() { |
|||
super.init(); |
|||
} |
|||
|
|||
@PreDestroy |
|||
@Override |
|||
public void close() { |
|||
super.close(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,187 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageDataIterable; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
|
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.CopyOnWriteArrayList; |
|||
import java.util.concurrent.locks.Lock; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldCache implements CalculatedFieldCache { |
|||
|
|||
private static final Integer UNKNOWN_PARTITION = -1; |
|||
|
|||
private final Lock calculatedFieldFetchLock = new ReentrantLock(); |
|||
|
|||
private final CalculatedFieldService calculatedFieldService; |
|||
private final TbelInvokeService tbelInvokeService; |
|||
private final ActorSystemContext actorSystemContext; |
|||
private final ApiLimitService apiLimitService; |
|||
|
|||
private final ConcurrentMap<CalculatedFieldId, CalculatedField> calculatedFields = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<EntityId, List<CalculatedField>> entityIdCalculatedFields = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<CalculatedFieldId, List<CalculatedFieldLink>> calculatedFieldLinks = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<EntityId, List<CalculatedFieldLink>> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); |
|||
private final ConcurrentMap<CalculatedFieldId, CalculatedFieldCtx> calculatedFieldsCtx = new ConcurrentHashMap<>(); |
|||
|
|||
@Value("${calculatedField.initFetchPackSize:50000}") |
|||
@Getter |
|||
private int initFetchPackSize; |
|||
|
|||
@AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) |
|||
public void init() { |
|||
//TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService)
|
|||
PageDataIterable<CalculatedField> cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); |
|||
cfs.forEach(cf -> { |
|||
calculatedFields.putIfAbsent(cf.getId(), cf); |
|||
actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); |
|||
}); |
|||
calculatedFields.values().forEach(cf -> { |
|||
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); |
|||
}); |
|||
PageDataIterable<CalculatedFieldLink> cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); |
|||
cfls.forEach(link -> { |
|||
calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link); |
|||
actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); |
|||
}); |
|||
calculatedFieldLinks.values().stream() |
|||
.flatMap(List::stream) |
|||
.forEach(link -> |
|||
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) |
|||
); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { |
|||
return calculatedFields.get(calculatedFieldId); |
|||
} |
|||
|
|||
@Override |
|||
public List<CalculatedField> getCalculatedFieldsByEntityId(EntityId entityId) { |
|||
return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); |
|||
} |
|||
|
|||
@Override |
|||
public List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId) { |
|||
return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { |
|||
CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); |
|||
if (ctx == null) { |
|||
calculatedFieldFetchLock.lock(); |
|||
try { |
|||
ctx = calculatedFieldsCtx.get(calculatedFieldId); |
|||
if (ctx == null) { |
|||
CalculatedField calculatedField = getCalculatedField(calculatedFieldId); |
|||
if (calculatedField != null) { |
|||
ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); |
|||
calculatedFieldsCtx.put(calculatedFieldId, ctx); |
|||
log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); |
|||
} |
|||
} |
|||
} finally { |
|||
calculatedFieldFetchLock.unlock(); |
|||
} |
|||
} |
|||
log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); |
|||
return ctx; |
|||
} |
|||
|
|||
@Override |
|||
public List<CalculatedFieldCtx> getCalculatedFieldCtxsByEntityId(EntityId entityId) { |
|||
if (entityId == null) { |
|||
return Collections.emptyList(); |
|||
} |
|||
return getCalculatedFieldsByEntityId(entityId).stream() |
|||
.map(cf -> getCalculatedFieldCtx(cf.getId())) |
|||
.toList(); |
|||
} |
|||
|
|||
@Override |
|||
public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { |
|||
calculatedFieldFetchLock.lock(); |
|||
try { |
|||
CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); |
|||
EntityId cfEntityId = calculatedField.getEntityId(); |
|||
|
|||
calculatedFields.put(calculatedFieldId, calculatedField); |
|||
|
|||
entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); |
|||
|
|||
CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); |
|||
calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); |
|||
|
|||
configuration.getReferencedEntities().stream() |
|||
.filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) |
|||
.forEach(referencedEntityId -> { |
|||
entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) |
|||
.add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); |
|||
}); |
|||
} finally { |
|||
calculatedFieldFetchLock.unlock(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { |
|||
evict(calculatedFieldId); |
|||
addCalculatedField(tenantId, calculatedFieldId); |
|||
} |
|||
|
|||
@Override |
|||
public void evict(CalculatedFieldId calculatedFieldId) { |
|||
CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); |
|||
log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); |
|||
calculatedFieldLinks.remove(calculatedFieldId); |
|||
log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); |
|||
entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); |
|||
log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); |
|||
calculatedFieldsCtx.remove(calculatedFieldId); |
|||
log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); |
|||
entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); |
|||
log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,59 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.ProfileEntityIdInfo; |
|||
import org.thingsboard.server.common.data.page.PageDataIterable; |
|||
import org.thingsboard.server.dao.asset.AssetService; |
|||
import org.thingsboard.server.dao.device.DeviceService; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; |
|||
|
|||
@Slf4j |
|||
@Service |
|||
@TbRuleEngineComponent |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { |
|||
|
|||
private final CalculatedFieldEntityProfileCache entityProfileCache; |
|||
private final AssetService assetService; |
|||
private final DeviceService deviceService; |
|||
|
|||
@Value("${calculated_fields.init_fetch_pack_size:50000}") |
|||
@Getter |
|||
private int initFetchPackSize; |
|||
|
|||
@AfterStartUp(order = AfterStartUp.CF_READ_PROFILE_ENTITIES_SERVICE) |
|||
public void initCalculatedFieldDefinitions() { |
|||
PageDataIterable<ProfileEntityIdInfo> deviceIdInfos = new PageDataIterable<>(deviceService::findProfileEntityIdInfos, initFetchPackSize); |
|||
for (ProfileEntityIdInfo idInfo : deviceIdInfos) { |
|||
log.trace("Processing device record: {}", idInfo); |
|||
entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); |
|||
} |
|||
PageDataIterable<ProfileEntityIdInfo> assetIdInfos = new PageDataIterable<>(assetService::findProfileEntityIdInfos, initFetchPackSize); |
|||
for (ProfileEntityIdInfo idInfo : assetIdInfos) { |
|||
log.trace("Processing asset record: {}", idInfo); |
|||
entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,325 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.ListeningExecutorService; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.math.NumberUtils; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.common.util.ThingsBoardExecutors; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
|||
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; |
|||
import org.thingsboard.server.cluster.TbClusterService; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.StringUtils; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.Aggregation; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BooleanDataEntry; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; |
|||
import org.thingsboard.server.common.msg.TbMsg; |
|||
import org.thingsboard.server.common.msg.TbMsgMetaData; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; |
|||
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; |
|||
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Map.Entry; |
|||
import java.util.Optional; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutionException; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.thingsboard.server.common.data.DataConstants.SCOPE; |
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
|||
|
|||
@TbRuleEngineComponent |
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { |
|||
|
|||
private final AttributesService attributesService; |
|||
private final TimeseriesService timeseriesService; |
|||
private final TbClusterService clusterService; |
|||
private final ApiLimitService apiLimitService; |
|||
private final PartitionService partitionService; |
|||
|
|||
private ListeningExecutorService calculatedFieldCallbackExecutor; |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( |
|||
Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); |
|||
} |
|||
|
|||
@PreDestroy |
|||
public void stop() { |
|||
if (calculatedFieldCallbackExecutor != null) { |
|||
calculatedFieldCallbackExecutor.shutdownNow(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<CalculatedFieldState> fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { |
|||
Map<String, ListenableFuture<ArgumentEntry>> argFutures = new HashMap<>(); |
|||
for (var entry : ctx.getArguments().entrySet()) { |
|||
var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; |
|||
var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); |
|||
argFutures.put(entry.getKey(), argValueFuture); |
|||
} |
|||
return Futures.whenAllComplete(argFutures.values()).call(() -> { |
|||
var result = createStateByType(ctx); |
|||
result.updateState(ctx, argFutures.entrySet().stream() |
|||
.collect(Collectors.toMap( |
|||
Entry::getKey, // Keep the key as is
|
|||
entry -> { |
|||
try { |
|||
// Resolve the future to get the value
|
|||
return entry.getValue().get(); |
|||
} catch (ExecutionException | InterruptedException e) { |
|||
throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); |
|||
} |
|||
} |
|||
))); |
|||
return result; |
|||
}, calculatedFieldCallbackExecutor); |
|||
} |
|||
|
|||
@Override |
|||
public Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments) { |
|||
Map<String, ListenableFuture<ArgumentEntry>> argFutures = new HashMap<>(); |
|||
for (var entry : arguments.entrySet()) { |
|||
var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; |
|||
var argValueFuture = fetchKvEntry(tenantId, argEntityId, entry.getValue()); |
|||
argFutures.put(entry.getKey(), argValueFuture); |
|||
} |
|||
return argFutures.entrySet().stream() |
|||
.collect(Collectors.toMap( |
|||
Entry::getKey, // Keep the key as is
|
|||
entry -> { |
|||
try { |
|||
// Resolve the future to get the value
|
|||
return entry.getValue().get(); |
|||
} catch (ExecutionException | InterruptedException e) { |
|||
throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); |
|||
} |
|||
} |
|||
)); |
|||
} |
|||
|
|||
@Override |
|||
public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List<CalculatedFieldId> cfIds, TbCallback callback) { |
|||
try { |
|||
OutputType type = calculatedFieldResult.getType(); |
|||
TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; |
|||
TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; |
|||
TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); |
|||
clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
callback.onSuccess(); |
|||
log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(t); |
|||
} |
|||
}); |
|||
} catch (Exception e) { |
|||
log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); |
|||
callback.onFailure(e); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback) { |
|||
Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts = new HashMap<>(); |
|||
List<CalculatedFieldEntityCtxId> broadcasts = new ArrayList<>(); |
|||
for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { |
|||
var linkEntityId = link.entityId(); |
|||
var linkEntityType = linkEntityId.getEntityType(); |
|||
// Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P.
|
|||
boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); |
|||
if (broadcast) { |
|||
broadcasts.add(link); |
|||
} else { |
|||
TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); |
|||
unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); |
|||
} |
|||
} |
|||
MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); |
|||
if (!broadcasts.isEmpty()) { |
|||
broadcast(broadcasts, msg, linkCallback); |
|||
} else { |
|||
linkCallback.onSuccess(); |
|||
} |
|||
if (!unicasts.isEmpty()) { |
|||
unicast(unicasts, msg, linkCallback); |
|||
} else { |
|||
linkCallback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
private void unicast(Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { |
|||
TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); |
|||
unicasts.forEach((topicPartitionInfo, ctxIds) -> { |
|||
CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); |
|||
clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), |
|||
ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); |
|||
}); |
|||
} |
|||
|
|||
private void broadcast(List<CalculatedFieldEntityCtxId> broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { |
|||
TbQueueCallback callback = new TbCallbackWrapper(mainCallback); |
|||
CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); |
|||
clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); |
|||
} |
|||
|
|||
private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List<CalculatedFieldEntityCtxId> links) { |
|||
Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); |
|||
builder.setMsg(telemetryProto); |
|||
for (CalculatedFieldEntityCtxId link : links) { |
|||
builder.addLinks(toProto(link)); |
|||
} |
|||
return builder.build(); |
|||
} |
|||
|
|||
private ListenableFuture<ArgumentEntry> fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { |
|||
return switch (argument.getRefEntityKey().getType()) { |
|||
case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); |
|||
case ATTRIBUTE -> transformSingleValueArgument( |
|||
Futures.transform( |
|||
attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), |
|||
result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), |
|||
calculatedFieldCallbackExecutor) |
|||
); |
|||
case TS_LATEST -> transformSingleValueArgument( |
|||
Futures.transform( |
|||
timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), |
|||
result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), |
|||
calculatedFieldCallbackExecutor)); |
|||
}; |
|||
} |
|||
|
|||
private ListenableFuture<ArgumentEntry> transformSingleValueArgument(ListenableFuture<Optional<? extends KvEntry>> kvEntryFuture) { |
|||
return Futures.transform(kvEntryFuture, kvEntry -> { |
|||
if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { |
|||
return ArgumentEntry.createSingleValueArgument(kvEntry.get()); |
|||
} else { |
|||
return new SingleValueArgumentEntry(); |
|||
} |
|||
}, calculatedFieldCallbackExecutor); |
|||
} |
|||
|
|||
private ListenableFuture<ArgumentEntry> fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { |
|||
long currentTime = System.currentTimeMillis(); |
|||
long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); |
|||
long startTs = currentTime - timeWindow; |
|||
long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); |
|||
int argumentLimit = argument.getLimit(); |
|||
int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argument.getLimit(); |
|||
|
|||
ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); |
|||
ListenableFuture<List<TsKvEntry>> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); |
|||
|
|||
return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? new TsRollingArgumentEntry(limit, timeWindow) : ArgumentEntry.createTsRollingArgument(tsRolling, limit, timeWindow), calculatedFieldCallbackExecutor); |
|||
} |
|||
|
|||
private KvEntry createDefaultKvEntry(Argument argument) { |
|||
String key = argument.getRefEntityKey().getKey(); |
|||
String defaultValue = argument.getDefaultValue(); |
|||
if (StringUtils.isBlank(defaultValue)) { |
|||
return new StringDataEntry(key, null); |
|||
} |
|||
if (NumberUtils.isParsable(defaultValue)) { |
|||
return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); |
|||
} |
|||
if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { |
|||
return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); |
|||
} |
|||
return new StringDataEntry(key, defaultValue); |
|||
} |
|||
|
|||
private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { |
|||
return switch (ctx.getCfType()) { |
|||
case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); |
|||
case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); |
|||
}; |
|||
} |
|||
|
|||
private static class TbCallbackWrapper implements TbQueueCallback { |
|||
private final TbCallback callback; |
|||
|
|||
public TbCallbackWrapper(TbCallback callback) { |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
callback.onSuccess(); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,275 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf; |
|||
|
|||
import com.google.common.util.concurrent.FutureCallback; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.rule.engine.api.AttributesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.AttributesSaveRequest; |
|||
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; |
|||
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; |
|||
import org.thingsboard.server.cluster.TbClusterService; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
|||
import org.thingsboard.server.common.data.id.AssetId; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.msg.TbMsgType; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
|||
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
|||
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
|||
|
|||
import java.util.Collections; |
|||
import java.util.EnumSet; |
|||
import java.util.List; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.function.Predicate; |
|||
import java.util.function.Supplier; |
|||
|
|||
import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; |
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { |
|||
|
|||
public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
} |
|||
}; |
|||
|
|||
private final TbAssetProfileCache assetProfileCache; |
|||
private final TbDeviceProfileCache deviceProfileCache; |
|||
private final CalculatedFieldCache calculatedFieldCache; |
|||
private final TbClusterService clusterService; |
|||
|
|||
private static final Set<EntityType> supportedReferencedEntities = EnumSet.of( |
|||
EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT |
|||
); |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback<Void> callback) { |
|||
pushRequestToQueue(request, null, callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(AttributesSaveRequest request, FutureCallback<Void> callback) { |
|||
pushRequestToQueue(request, null, callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
@Override |
|||
public void pushRequestToQueue(TimeseriesDeleteRequest request, List<String> result, FutureCallback<Void> callback) { |
|||
var tenantId = request.getTenantId(); |
|||
var entityId = request.getEntityId(); |
|||
|
|||
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), |
|||
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
|||
} |
|||
|
|||
private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, |
|||
Predicate<CalculatedFieldCtx> mainEntityFilter, Predicate<CalculatedFieldCtx> linkedEntityFilter, |
|||
Supplier<ToCalculatedFieldMsg> msg, FutureCallback<Void> callback) { |
|||
boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); |
|||
if (send) { |
|||
clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); |
|||
} else { |
|||
if (callback != null) { |
|||
callback.onSuccess(null); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate<CalculatedFieldCtx> filter, Predicate<CalculatedFieldCtx> linkedEntityFilter) { |
|||
boolean send = false; |
|||
if (supportedReferencedEntities.contains(entityId.getEntityType())) { |
|||
send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); |
|||
if (!send) { |
|||
send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); |
|||
} |
|||
if (!send) { |
|||
send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() |
|||
.map(CalculatedFieldLink::getCalculatedFieldId) |
|||
.map(calculatedFieldCache::getCalculatedFieldCtx) |
|||
.anyMatch(linkedEntityFilter); |
|||
} |
|||
} |
|||
return send; |
|||
} |
|||
|
|||
private EntityId getProfileId(TenantId tenantId, EntityId entityId) { |
|||
return switch (entityId.getEntityType()) { |
|||
case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); |
|||
case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); |
|||
default -> null; |
|||
}; |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { |
|||
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); |
|||
|
|||
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); |
|||
|
|||
List<TsKvEntry> entries = request.getEntries(); |
|||
List<Long> versions = result != null ? result.getVersions() : Collections.emptyList(); |
|||
|
|||
for (int i = 0; i < entries.size(); i++) { |
|||
TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); |
|||
if (result != null) { |
|||
tsProtoBuilder.setVersion(versions.get(i)); |
|||
} |
|||
telemetryMsg.addTsData(tsProtoBuilder.build()); |
|||
} |
|||
|
|||
msg.setTelemetryMsg(telemetryMsg.build()); |
|||
return msg.build(); |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List<Long> versions) { |
|||
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); |
|||
|
|||
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); |
|||
telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); |
|||
List<AttributeKvEntry> entries = request.getEntries(); |
|||
for (int i = 0; i < entries.size(); i++) { |
|||
AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder(); |
|||
if (versions != null) { |
|||
attrProtoBuilder.setVersion(versions.get(i)); |
|||
} |
|||
telemetryMsg.addAttrData(attrProtoBuilder.build()); |
|||
} |
|||
msg.setTelemetryMsg(telemetryMsg.build()); |
|||
|
|||
return msg.build(); |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesDeleteRequest request, List<String> removedKeys) { |
|||
CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) |
|||
.setScope(AttributeScopeProto.valueOf(request.getScope().name())) |
|||
.addAllRemovedAttrKeys(removedKeys).build(); |
|||
return ToCalculatedFieldMsg.newBuilder() |
|||
.setTelemetryMsg(telemetryMsg) |
|||
.build(); |
|||
} |
|||
|
|||
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesDeleteRequest request, List<String> removedKeys) { |
|||
CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) |
|||
.addAllRemovedTsKeys(removedKeys).build(); |
|||
return ToCalculatedFieldMsg.newBuilder() |
|||
.setTelemetryMsg(telemetryMsg) |
|||
.build(); |
|||
} |
|||
|
|||
private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List<CalculatedFieldId> calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { |
|||
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); |
|||
|
|||
telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); |
|||
telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); |
|||
|
|||
telemetryMsg.setEntityType(entityId.getEntityType().name()); |
|||
telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); |
|||
telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); |
|||
|
|||
if (calculatedFieldIds != null) { |
|||
for (CalculatedFieldId cfId : calculatedFieldIds) { |
|||
telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); |
|||
} |
|||
} |
|||
|
|||
if (tbMsgId != null) { |
|||
telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); |
|||
telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); |
|||
} |
|||
|
|||
if (tbMsgType != null) { |
|||
telemetryMsg.setTbMsgType(tbMsgType.name()); |
|||
} |
|||
|
|||
return telemetryMsg; |
|||
} |
|||
|
|||
private static TbQueueCallback wrap(FutureCallback<Void> callback) { |
|||
if (callback != null) { |
|||
return new FutureCallbackWrapper(callback); |
|||
} else { |
|||
return DUMMY_TB_QUEUE_CALLBACK; |
|||
} |
|||
} |
|||
|
|||
private static class FutureCallbackWrapper implements TbQueueCallback { |
|||
private final FutureCallback<Void> callback; |
|||
|
|||
public FutureCallbackWrapper(FutureCallback<Void> callback) { |
|||
this.callback = callback; |
|||
} |
|||
|
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
callback.onSuccess(null); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.cache; |
|||
|
|||
import org.springframework.context.ApplicationListener; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
|
|||
import java.util.Collection; |
|||
|
|||
public interface CalculatedFieldEntityProfileCache extends ApplicationListener<PartitionChangeEvent> { |
|||
|
|||
void add(TenantId tenantId, EntityId profileId, EntityId entityId); |
|||
|
|||
void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); |
|||
|
|||
void evict(TenantId tenantId, EntityId entityId); |
|||
|
|||
Collection<EntityId> getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); |
|||
|
|||
int getEntityIdPartition(TenantId tenantId, EntityId entityId); |
|||
} |
|||
@ -0,0 +1,93 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.cache; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.discovery.TbApplicationEventListener; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
|
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@TbRuleEngineComponent |
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
//TODO ashvayka: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor;
|
|||
public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener<PartitionChangeEvent> implements CalculatedFieldEntityProfileCache { |
|||
|
|||
private static final Integer UNKNOWN = 0; |
|||
private final ConcurrentMap<TenantId, TenantEntityProfileCache> tenantCache = new ConcurrentHashMap<>(); |
|||
private final PartitionService partitionService; |
|||
private volatile List<Integer> myPartitions = Collections.emptyList(); |
|||
|
|||
@Override |
|||
protected void onTbApplicationEvent(PartitionChangeEvent event) { |
|||
myPartitions = event.getCfPartitions().stream() |
|||
.filter(TopicPartitionInfo::isMyPartition) |
|||
.map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); |
|||
//Naive approach that need to be improved.
|
|||
tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); |
|||
} |
|||
|
|||
@Override |
|||
public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { |
|||
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
|||
var partition = tpi.getPartition().orElse(UNKNOWN); |
|||
tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) |
|||
.add(profileId, entityId, partition, tpi.isMyPartition()); |
|||
} |
|||
|
|||
@Override |
|||
public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { |
|||
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
|||
var partition = tpi.getPartition().orElse(UNKNOWN); |
|||
var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); |
|||
//TODO: make this method atomic;
|
|||
cache.remove(oldProfileId, entityId); |
|||
cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); |
|||
} |
|||
|
|||
@Override |
|||
public void evict(TenantId tenantId, EntityId entityId) { |
|||
var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); |
|||
cache.removeEntityId(entityId); |
|||
} |
|||
|
|||
@Override |
|||
public Collection<EntityId> getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { |
|||
return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); |
|||
} |
|||
|
|||
@Override |
|||
public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { |
|||
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
|||
return tpi.getPartition().orElse(UNKNOWN); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,122 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.cache; |
|||
|
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Collection; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.HashSet; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.concurrent.locks.ReadWriteLock; |
|||
import java.util.concurrent.locks.ReentrantReadWriteLock; |
|||
|
|||
public class TenantEntityProfileCache { |
|||
|
|||
private final ReadWriteLock lock = new ReentrantReadWriteLock(); |
|||
private final Map<Integer, Map<EntityId, Set<EntityId>>> allEntities = new HashMap<>(); |
|||
private final Map<EntityId, Set<EntityId>> myEntities = new HashMap<>(); |
|||
|
|||
public void setMyPartitions(List<Integer> myPartitions) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
myEntities.clear(); |
|||
myPartitions.forEach(partitionId -> { |
|||
var map = allEntities.get(partitionId); |
|||
if (map != null) { |
|||
map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); |
|||
} |
|||
}); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void removeProfileId(EntityId profileId) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
// Remove from allEntities
|
|||
allEntities.values().forEach(map -> map.remove(profileId)); |
|||
// Remove from myEntities
|
|||
myEntities.remove(profileId); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void removeEntityId(EntityId entityId) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
// Remove from allEntities
|
|||
allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); |
|||
// Remove from myEntities
|
|||
myEntities.values().forEach(set -> set.remove(entityId)); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void remove(EntityId profileId, EntityId entityId) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
// Remove from allEntities
|
|||
allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); |
|||
// Remove from myEntities
|
|||
removeSafely(myEntities, profileId, entityId); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { |
|||
lock.writeLock().lock(); |
|||
try { |
|||
if(EntityType.DEVICE.equals(profileId.getEntityType())){ |
|||
throw new RuntimeException("WTF?"); |
|||
} |
|||
if (mine) { |
|||
myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); |
|||
} |
|||
allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); |
|||
} finally { |
|||
lock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
public Collection<EntityId> getMyEntityIdsByProfileId(EntityId profileId) { |
|||
lock.readLock().lock(); |
|||
try { |
|||
var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); |
|||
List<EntityId> result = new ArrayList<>(entities.size()); |
|||
result.addAll(entities); |
|||
return result; |
|||
} finally { |
|||
lock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
private void removeSafely(Map<EntityId, Set<EntityId>> map, EntityId profileId, EntityId entityId) { |
|||
var set = map.get(profileId); |
|||
if (set != null) { |
|||
set.remove(entityId); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx; |
|||
|
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
public class CalculatedFieldEntityCtx { |
|||
|
|||
private CalculatedFieldEntityCtxId id; |
|||
private CalculatedFieldState state; |
|||
|
|||
public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { |
|||
this.id = id; |
|||
this.state = state; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,28 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx; |
|||
|
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { |
|||
|
|||
public String toKey() { |
|||
return cfId + "_" + entityId; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,61 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import com.fasterxml.jackson.annotation.JsonSubTypes; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
|
|||
import java.util.List; |
|||
|
|||
@JsonTypeInfo( |
|||
use = JsonTypeInfo.Id.NAME, |
|||
include = JsonTypeInfo.As.PROPERTY, |
|||
property = "type" |
|||
) |
|||
@JsonSubTypes({ |
|||
@JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), |
|||
@JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") |
|||
}) |
|||
public interface ArgumentEntry { |
|||
|
|||
@JsonIgnore |
|||
ArgumentEntryType getType(); |
|||
|
|||
Object getValue(); |
|||
|
|||
boolean updateEntry(ArgumentEntry entry); |
|||
|
|||
boolean isEmpty(); |
|||
|
|||
TbelCfArg toTbelCfArg(); |
|||
|
|||
boolean isForceResetPrevious(); |
|||
|
|||
void setForceResetPrevious(boolean forceResetPrevious); |
|||
|
|||
static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { |
|||
return new SingleValueArgumentEntry(kvEntry); |
|||
} |
|||
|
|||
static ArgumentEntry createTsRollingArgument(List<TsKvEntry> kvEntries, int limit, long timeWindow) { |
|||
return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,20 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
public enum ArgumentEntryType { |
|||
SINGLE_VALUE, TS_ROLLING |
|||
} |
|||
@ -0,0 +1,103 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
import org.thingsboard.server.utils.CalculatedFieldUtils; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; |
|||
|
|||
@Data |
|||
@AllArgsConstructor |
|||
public abstract class BaseCalculatedFieldState implements CalculatedFieldState { |
|||
|
|||
protected List<String> requiredArguments; |
|||
protected Map<String, ArgumentEntry> arguments; |
|||
protected boolean sizeExceedsLimit; |
|||
|
|||
public BaseCalculatedFieldState(List<String> requiredArguments) { |
|||
this.requiredArguments = requiredArguments; |
|||
this.arguments = new HashMap<>(); |
|||
} |
|||
|
|||
public BaseCalculatedFieldState() { |
|||
this(new ArrayList<>(), new HashMap<>(), false); |
|||
} |
|||
|
|||
@Override |
|||
public boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues) { |
|||
if (arguments == null) { |
|||
arguments = new HashMap<>(); |
|||
} |
|||
|
|||
boolean stateUpdated = false; |
|||
|
|||
for (Map.Entry<String, ArgumentEntry> entry : argumentValues.entrySet()) { |
|||
String key = entry.getKey(); |
|||
ArgumentEntry newEntry = entry.getValue(); |
|||
|
|||
checkArgumentSize(key, newEntry, ctx); |
|||
|
|||
ArgumentEntry existingEntry = arguments.get(key); |
|||
|
|||
if (existingEntry == null || newEntry.isForceResetPrevious()) { |
|||
validateNewEntry(newEntry); |
|||
arguments.put(key, newEntry); |
|||
stateUpdated = true; |
|||
} else { |
|||
stateUpdated = existingEntry.updateEntry(newEntry); |
|||
} |
|||
} |
|||
|
|||
return stateUpdated; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isReady() { |
|||
return arguments.keySet().containsAll(requiredArguments) && |
|||
arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); |
|||
} |
|||
|
|||
@Override |
|||
public void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize) { |
|||
if (!sizeExceedsLimit && maxStateSize > 0 && CalculatedFieldUtils.toProto(ctxId, this).getSerializedSize() > maxStateSize) { |
|||
arguments.clear(); |
|||
sizeExceedsLimit = true; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx) { |
|||
if (entry instanceof TsRollingArgumentEntry) { |
|||
return; |
|||
} |
|||
if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { |
|||
if (ctx.getMaxSingleValueArgumentSize() > 0 && toSingleValueArgumentProto(name, singleValueArgumentEntry).getSerializedSize() > ctx.getMaxSingleValueArgumentSize()) { |
|||
throw new IllegalArgumentException("Single value size exceeds the maximum allowed limit. The argument will not be used for calculation."); |
|||
} |
|||
} |
|||
} |
|||
|
|||
protected abstract void validateNewEntry(ArgumentEntry newEntry); |
|||
|
|||
} |
|||
@ -0,0 +1,282 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import lombok.Data; |
|||
import net.objecthunter.exp4j.Expression; |
|||
import net.objecthunter.exp4j.ExpressionBuilder; |
|||
import org.mvel2.MVEL; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Argument; |
|||
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
|||
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; |
|||
import org.thingsboard.server.common.data.util.TbPair; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Data |
|||
public class CalculatedFieldCtx { |
|||
|
|||
private CalculatedField calculatedField; |
|||
|
|||
private CalculatedFieldId cfId; |
|||
private TenantId tenantId; |
|||
private EntityId entityId; |
|||
private CalculatedFieldType cfType; |
|||
private final Map<String, Argument> arguments; |
|||
private final Map<ReferencedEntityKey, String> mainEntityArguments; |
|||
private final Map<EntityId, Map<ReferencedEntityKey, String>> linkedEntityArguments; |
|||
|
|||
private final Map<TbPair<EntityId, ReferencedEntityKey>, String> referencedEntityKeys; |
|||
private final List<String> argNames; |
|||
private Output output; |
|||
private String expression; |
|||
private TbelInvokeService tbelInvokeService; |
|||
private CalculatedFieldScriptEngine calculatedFieldScriptEngine; |
|||
private ThreadLocal<Expression> customExpression; |
|||
|
|||
private boolean initialized; |
|||
|
|||
private long maxDataPointsPerRollingArg; |
|||
private long maxStateSize; |
|||
private long maxSingleValueArgumentSize; |
|||
|
|||
public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { |
|||
this.calculatedField = calculatedField; |
|||
|
|||
this.cfId = calculatedField.getId(); |
|||
this.tenantId = calculatedField.getTenantId(); |
|||
this.entityId = calculatedField.getEntityId(); |
|||
this.cfType = calculatedField.getType(); |
|||
CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); |
|||
this.arguments = configuration.getArguments(); |
|||
this.mainEntityArguments = new HashMap<>(); |
|||
this.linkedEntityArguments = new HashMap<>(); |
|||
for (Map.Entry<String, Argument> entry : arguments.entrySet()) { |
|||
var refId = entry.getValue().getRefEntityId(); |
|||
var refKey = entry.getValue().getRefEntityKey(); |
|||
if (refId == null) { |
|||
mainEntityArguments.put(refKey, entry.getKey()); |
|||
} else { |
|||
linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); |
|||
} |
|||
} |
|||
this.referencedEntityKeys = arguments.entrySet().stream() |
|||
.collect(Collectors.toMap( |
|||
entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), |
|||
Map.Entry::getKey |
|||
)); |
|||
this.argNames = new ArrayList<>(arguments.keySet()); |
|||
this.output = configuration.getOutput(); |
|||
this.expression = configuration.getExpression(); |
|||
this.tbelInvokeService = tbelInvokeService; |
|||
|
|||
this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); |
|||
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; |
|||
this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; |
|||
} |
|||
|
|||
public void init() { |
|||
if (CalculatedFieldType.SCRIPT.equals(cfType)) { |
|||
try { |
|||
this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); |
|||
initialized = true; |
|||
} catch (Exception e) { |
|||
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); |
|||
} |
|||
} else { |
|||
if (isValidExpression(expression)) { |
|||
this.customExpression = ThreadLocal.withInitial(() -> |
|||
new ExpressionBuilder(expression) |
|||
.implicitMultiplication(true) |
|||
.variables(this.arguments.keySet()) |
|||
.build() |
|||
); |
|||
initialized = true; |
|||
} else { |
|||
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { |
|||
if (tbelInvokeService == null) { |
|||
throw new IllegalArgumentException("TBEL script engine is disabled!"); |
|||
} |
|||
|
|||
List<String> ctxAndArgNames = new ArrayList<>(argNames.size() + 1); |
|||
ctxAndArgNames.add("ctx"); |
|||
ctxAndArgNames.addAll(argNames); |
|||
return new CalculatedFieldTbelScriptEngine( |
|||
tenantId, |
|||
tbelInvokeService, |
|||
expression, |
|||
ctxAndArgNames.toArray(String[]::new) |
|||
); |
|||
} |
|||
|
|||
private boolean isValidExpression(String expression) { |
|||
try { |
|||
MVEL.compileExpression(expression); |
|||
return true; |
|||
} catch (Exception e) { |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
public boolean matches(List<AttributeKvEntry> values, AttributeScope scope) { |
|||
return matchesAttributes(mainEntityArguments, values, scope); |
|||
} |
|||
|
|||
public boolean linkMatches(EntityId entityId, List<AttributeKvEntry> values, AttributeScope scope) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesAttributes(map, values, scope); |
|||
} |
|||
|
|||
public boolean matches(List<TsKvEntry> values) { |
|||
return matchesTimeSeries(mainEntityArguments, values); |
|||
} |
|||
|
|||
public boolean linkMatches(EntityId entityId, List<TsKvEntry> values) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesTimeSeries(map, values); |
|||
} |
|||
|
|||
private boolean matchesAttributes(Map<ReferencedEntityKey, String> argMap, List<AttributeKvEntry> values, AttributeScope scope) { |
|||
for (AttributeKvEntry attrKv : values) { |
|||
ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); |
|||
if (argMap.containsKey(attrKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
private boolean matchesTimeSeries(Map<ReferencedEntityKey, String> argMap, List<TsKvEntry> values) { |
|||
for (TsKvEntry tsKv : values) { |
|||
ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); |
|||
if (argMap.containsKey(latestKey)) { |
|||
return true; |
|||
} |
|||
ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); |
|||
if (argMap.containsKey(rollingKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
public boolean matchesKeys(List<String> keys, AttributeScope scope) { |
|||
return matchesAttributesKeys(mainEntityArguments, keys, scope); |
|||
} |
|||
|
|||
public boolean matchesKeys(List<String> keys) { |
|||
return matchesTimeSeriesKeys(mainEntityArguments, keys); |
|||
} |
|||
|
|||
private boolean matchesAttributesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys, AttributeScope scope) { |
|||
for (String key : keys) { |
|||
ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); |
|||
if (argMap.containsKey(attrKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
private boolean matchesTimeSeriesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys) { |
|||
for (String key : keys) { |
|||
ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); |
|||
if (argMap.containsKey(latestKey)) { |
|||
return true; |
|||
} |
|||
ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); |
|||
if (argMap.containsKey(rollingKey)) { |
|||
return true; |
|||
} |
|||
} |
|||
return false; |
|||
} |
|||
|
|||
public boolean linkMatchesAttrKeys(EntityId entityId, List<String> keys, AttributeScope scope) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesAttributesKeys(map, keys, scope); |
|||
} |
|||
|
|||
public boolean linkMatchesTsKeys(EntityId entityId, List<String> keys) { |
|||
var map = linkedEntityArguments.get(entityId); |
|||
return map != null && matchesTimeSeriesKeys(map, keys); |
|||
} |
|||
|
|||
public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { |
|||
if (!proto.getTsDataList().isEmpty()) { |
|||
List<TsKvEntry> updatedTelemetry = proto.getTsDataList().stream() |
|||
.map(ProtoUtils::fromProto) |
|||
.toList(); |
|||
return linkMatches(entityId, updatedTelemetry); |
|||
} else if (!proto.getAttrDataList().isEmpty()) { |
|||
AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); |
|||
List<AttributeKvEntry> updatedTelemetry = proto.getAttrDataList().stream() |
|||
.map(ProtoUtils::fromProto) |
|||
.toList(); |
|||
return linkMatches(entityId, updatedTelemetry, scope); |
|||
} else if (!proto.getRemovedTsKeysList().isEmpty()) { |
|||
return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); |
|||
} else { |
|||
return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); |
|||
} |
|||
} |
|||
|
|||
public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { |
|||
return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); |
|||
} |
|||
|
|||
public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { |
|||
boolean expressionChanged = !expression.equals(other.expression); |
|||
boolean outputChanged = !output.equals(other.output); |
|||
return expressionChanged || outputChanged; |
|||
} |
|||
|
|||
public boolean hasStateChanges(CalculatedFieldCtx other) { |
|||
boolean typeChanged = !cfType.equals(other.cfType); |
|||
boolean argumentsChanged = !arguments.equals(other.arguments); |
|||
return typeChanged || argumentsChanged; |
|||
} |
|||
|
|||
public String getSizeExceedsLimitMessage() { |
|||
return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
|
|||
public interface CalculatedFieldScriptEngine { |
|||
|
|||
ListenableFuture<Object> executeScriptAsync(Object[] args); |
|||
|
|||
ListenableFuture<JsonNode> executeJsonAsync(Object[] args); |
|||
|
|||
void destroy(); |
|||
|
|||
} |
|||
@ -0,0 +1,65 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import com.fasterxml.jackson.annotation.JsonSubTypes; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
@JsonTypeInfo( |
|||
use = JsonTypeInfo.Id.NAME, |
|||
include = JsonTypeInfo.As.PROPERTY, |
|||
property = "type" |
|||
) |
|||
@JsonSubTypes({ |
|||
@JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), |
|||
@JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), |
|||
}) |
|||
public interface CalculatedFieldState { |
|||
|
|||
@JsonIgnore |
|||
CalculatedFieldType getType(); |
|||
|
|||
Map<String, ArgumentEntry> getArguments(); |
|||
|
|||
void setRequiredArguments(List<String> requiredArguments); |
|||
|
|||
boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues); |
|||
|
|||
ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx); |
|||
|
|||
@JsonIgnore |
|||
boolean isReady(); |
|||
|
|||
boolean isSizeExceedsLimit(); |
|||
|
|||
@JsonIgnore |
|||
default boolean isSizeOk() { |
|||
return !isSizeExceedsLimit(); |
|||
} |
|||
|
|||
void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); |
|||
|
|||
void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx); |
|||
|
|||
} |
|||
@ -0,0 +1,82 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.script.api.ScriptType; |
|||
import org.thingsboard.script.api.tbel.TbelInvokeService; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
import javax.script.ScriptException; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutionException; |
|||
|
|||
@Slf4j |
|||
public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { |
|||
|
|||
private final TbelInvokeService tbelInvokeService; |
|||
|
|||
private final UUID scriptId; |
|||
private final TenantId tenantId; |
|||
|
|||
public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { |
|||
this.tenantId = tenantId; |
|||
this.tbelInvokeService = tbelInvokeService; |
|||
try { |
|||
this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); |
|||
} catch (Exception e) { |
|||
Throwable t = e; |
|||
if (e instanceof ExecutionException) { |
|||
t = e.getCause(); |
|||
} |
|||
throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Object> executeScriptAsync(Object[] args) { |
|||
log.trace("Executing script async, args {}", args); |
|||
return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), |
|||
o -> { |
|||
try { |
|||
return Futures.immediateFuture(o); |
|||
} catch (Exception e) { |
|||
if (e.getCause() instanceof ScriptException) { |
|||
return Futures.immediateFailedFuture(e.getCause()); |
|||
} else if (e.getCause() instanceof RuntimeException) { |
|||
return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); |
|||
} else { |
|||
return Futures.immediateFailedFuture(new ScriptException(e)); |
|||
} |
|||
} |
|||
}, MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<JsonNode> executeJsonAsync(Object[] args) { |
|||
return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
@Override |
|||
public void destroy() { |
|||
tbelInvokeService.release(this.scriptId); |
|||
} |
|||
} |
|||
@ -0,0 +1,157 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueMsgHeaders; |
|||
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.queue.common.consumer.QueueStateService; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; |
|||
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; |
|||
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.Set; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.*; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine') && '${queue.type:null}'=='kafka'") |
|||
public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { |
|||
|
|||
private final TbRuleEngineQueueFactory queueFactory; |
|||
private final PartitionService partitionService; |
|||
|
|||
@Value("${queue.calculated_fields.poll_interval:25}") |
|||
private long pollInterval; |
|||
|
|||
private PartitionedQueueConsumerManager<TbProtoQueueMsg<CalculatedFieldStateProto>> stateConsumer; |
|||
private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>> stateProducer; |
|||
private QueueStateService<TbProtoQueueMsg<ToCalculatedFieldMsg>, TbProtoQueueMsg<CalculatedFieldStateProto>> queueStateService; |
|||
|
|||
private final AtomicInteger counter = new AtomicInteger(); |
|||
|
|||
@Override |
|||
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer) { |
|||
super.init(eventConsumer); |
|||
this.stateConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<CalculatedFieldStateProto>>create() |
|||
.queueKey(QueueKey.CF_STATES) |
|||
.topic(partitionService.getTopic(QueueKey.CF_STATES)) |
|||
.pollInterval(pollInterval) |
|||
.msgPackProcessor((msgs, consumer, config) -> { |
|||
for (TbProtoQueueMsg<CalculatedFieldStateProto> msg : msgs) { |
|||
try { |
|||
if (msg.getValue() != null) { |
|||
processRestoredState(msg.getValue()); |
|||
} else { |
|||
processRestoredState(getStateId(msg.getHeaders()), null); |
|||
} |
|||
} catch (Throwable t) { |
|||
log.error("Failed to process state message: {}", msg, t); |
|||
} |
|||
|
|||
int processedMsgCount = counter.incrementAndGet(); |
|||
if (processedMsgCount % 10000 == 0) { |
|||
log.info("Processed {} calculated field state msgs", processedMsgCount); |
|||
} |
|||
} |
|||
}) |
|||
.consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) |
|||
.consumerExecutor(eventConsumer.getConsumerExecutor()) |
|||
.scheduler(eventConsumer.getScheduler()) |
|||
.taskExecutor(eventConsumer.getTaskExecutor()) |
|||
.build(); |
|||
this.stateProducer = (TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>>) queueFactory.createCalculatedFieldStateProducer(); |
|||
this.queueStateService = new QueueStateService<>(); |
|||
this.queueStateService.init(stateConsumer, super.eventConsumer); |
|||
} |
|||
|
|||
@Override |
|||
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { |
|||
TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); |
|||
TbProtoQueueMsg<CalculatedFieldStateProto> msg = new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto); |
|||
if (stateMsgProto == null) { |
|||
putStateId(msg.getHeaders(), stateId); |
|||
} |
|||
stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
if (callback != null) { |
|||
callback.onSuccess(); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
if (callback != null) { |
|||
callback.onFailure(t); |
|||
} |
|||
} |
|||
}); |
|||
} |
|||
|
|||
@Override |
|||
protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
|||
doPersist(stateId, null, callback); |
|||
} |
|||
|
|||
@Override |
|||
public void restore(Set<TopicPartitionInfo> partitions) { |
|||
queueStateService.update(partitions); |
|||
} |
|||
|
|||
private void putStateId(TbQueueMsgHeaders headers, CalculatedFieldEntityCtxId stateId) { |
|||
headers.put("tenantId", uuidToBytes(stateId.tenantId().getId())); |
|||
headers.put("cfId", uuidToBytes(stateId.cfId().getId())); |
|||
headers.put("entityId", uuidToBytes(stateId.entityId().getId())); |
|||
headers.put("entityType", stringToBytes(stateId.entityId().getEntityType().name())); |
|||
} |
|||
|
|||
private CalculatedFieldEntityCtxId getStateId(TbQueueMsgHeaders headers) { |
|||
TenantId tenantId = TenantId.fromUUID(bytesToUuid(headers.get("tenantId"))); |
|||
CalculatedFieldId cfId = new CalculatedFieldId(bytesToUuid(headers.get("cfId"))); |
|||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(bytesToString(headers.get("entityType")), bytesToUuid(headers.get("entityId"))); |
|||
return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); |
|||
} |
|||
|
|||
@Override |
|||
public void stop() { |
|||
stateConsumer.stop(); |
|||
stateConsumer.awaitStop(); |
|||
stateProducer.stop(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,73 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.google.protobuf.InvalidProtocolBufferException; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
|||
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; |
|||
import org.thingsboard.server.service.cf.CfRocksDb; |
|||
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
|||
|
|||
import java.util.Set; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") |
|||
public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { |
|||
|
|||
private final CfRocksDb cfRocksDb; |
|||
|
|||
private boolean initialized; |
|||
|
|||
@Override |
|||
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { |
|||
cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); |
|||
callback.onSuccess(); |
|||
} |
|||
|
|||
@Override |
|||
protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
|||
cfRocksDb.delete(stateId.toKey()); |
|||
callback.onSuccess(); |
|||
} |
|||
|
|||
@Override |
|||
public void restore(Set<TopicPartitionInfo> partitions) { |
|||
if (!this.initialized) { |
|||
cfRocksDb.forEach((key, value) -> { |
|||
try { |
|||
processRestoredState(CalculatedFieldStateProto.parseFrom(value)); |
|||
} catch (InvalidProtocolBufferException e) { |
|||
log.error("[{}] Failed to process restored state", key, e); |
|||
} |
|||
}); |
|||
this.initialized = true; |
|||
} |
|||
eventConsumer.update(partitions); |
|||
} |
|||
|
|||
@Override |
|||
public void stop() { |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,83 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.script.api.tbel.TbelCfCtx; |
|||
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.HashMap; |
|||
import java.util.LinkedHashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
@Data |
|||
@Slf4j |
|||
@NoArgsConstructor |
|||
public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { |
|||
|
|||
public ScriptCalculatedFieldState(List<String> requiredArguments) { |
|||
super(requiredArguments); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedFieldType getType() { |
|||
return CalculatedFieldType.SCRIPT; |
|||
} |
|||
|
|||
@Override |
|||
protected void validateNewEntry(ArgumentEntry newEntry) { |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { |
|||
Map<String, TbelCfArg> arguments = new LinkedHashMap<>(); |
|||
List<Object> args = new ArrayList<>(ctx.getArgNames().size() + 1); |
|||
args.add(new Object()); // first element is a ctx, but we will set it later;
|
|||
for (String argName : ctx.getArgNames()) { |
|||
var arg = toTbelArgument(argName); |
|||
arguments.put(argName, arg); |
|||
if (arg instanceof TbelCfSingleValueArg svArg) { |
|||
args.add(svArg.getValue()); |
|||
} else { |
|||
args.add(arg); |
|||
} |
|||
} |
|||
args.set(0, new TbelCfCtx(arguments)); |
|||
ListenableFuture<JsonNode> resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray()); |
|||
Output output = ctx.getOutput(); |
|||
return Futures.transform(resultFuture, |
|||
result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), |
|||
MoreExecutors.directExecutor() |
|||
); |
|||
} |
|||
|
|||
private TbelCfArg toTbelArgument(String key) { |
|||
return arguments.get(key).toTbelCfArg(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,83 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.script.api.tbel.TbUtils; |
|||
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
|||
import org.thingsboard.server.common.data.cf.configuration.Output; |
|||
import org.thingsboard.server.common.data.kv.BasicKvEntry; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { |
|||
|
|||
public SimpleCalculatedFieldState(List<String> requiredArguments) { |
|||
super(requiredArguments); |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedFieldType getType() { |
|||
return CalculatedFieldType.SIMPLE; |
|||
} |
|||
|
|||
@Override |
|||
protected void validateNewEntry(ArgumentEntry newEntry) { |
|||
if (newEntry instanceof TsRollingArgumentEntry) { |
|||
throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { |
|||
var expr = ctx.getCustomExpression().get(); |
|||
|
|||
for (Map.Entry<String, ArgumentEntry> entry : this.arguments.entrySet()) { |
|||
try { |
|||
BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); |
|||
expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); |
|||
} catch (NumberFormatException e) { |
|||
throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); |
|||
} |
|||
} |
|||
|
|||
double expressionResult = expr.evaluate(); |
|||
|
|||
Output output = ctx.getOutput(); |
|||
Object result; |
|||
Integer decimals = output.getDecimalsByDefault(); |
|||
if (decimals != null) { |
|||
if (decimals.equals(0)) { |
|||
result = TbUtils.toInt(expressionResult); |
|||
} else { |
|||
result = TbUtils.toFixed(expressionResult, decimals); |
|||
} |
|||
} else { |
|||
result = expressionResult; |
|||
} |
|||
|
|||
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result)))); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,115 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BasicKvEntry; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.util.ProtoUtils; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
@AllArgsConstructor |
|||
public class SingleValueArgumentEntry implements ArgumentEntry { |
|||
|
|||
private long ts; |
|||
private BasicKvEntry kvEntryValue; |
|||
private Long version; |
|||
|
|||
private boolean forceResetPrevious; |
|||
|
|||
public SingleValueArgumentEntry(TsKvProto entry) { |
|||
this.ts = entry.getTs(); |
|||
if (entry.hasVersion()) { |
|||
this.version = entry.getVersion(); |
|||
} |
|||
this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); |
|||
} |
|||
|
|||
public SingleValueArgumentEntry(AttributeValueProto entry) { |
|||
this.ts = entry.getLastUpdateTs(); |
|||
if (entry.hasVersion()) { |
|||
this.version = entry.getVersion(); |
|||
} |
|||
this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); |
|||
} |
|||
|
|||
public SingleValueArgumentEntry(KvEntry entry) { |
|||
if (entry instanceof TsKvEntry tsKvEntry) { |
|||
this.ts = tsKvEntry.getTs(); |
|||
this.version = tsKvEntry.getVersion(); |
|||
} else if (entry instanceof AttributeKvEntry attributeKvEntry) { |
|||
this.ts = attributeKvEntry.getLastUpdateTs(); |
|||
this.version = attributeKvEntry.getVersion(); |
|||
} |
|||
this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); |
|||
} |
|||
|
|||
public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { |
|||
this.ts = ts; |
|||
this.kvEntryValue = kvEntryValue; |
|||
this.version = version; |
|||
} |
|||
|
|||
@Override |
|||
public ArgumentEntryType getType() { |
|||
return ArgumentEntryType.SINGLE_VALUE; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isEmpty() { |
|||
return kvEntryValue == null; |
|||
} |
|||
|
|||
@JsonIgnore |
|||
public Object getValue() { |
|||
return isEmpty() ? null : kvEntryValue.getValue(); |
|||
} |
|||
|
|||
@Override |
|||
public TbelCfArg toTbelCfArg() { |
|||
return new TbelCfSingleValueArg(ts, kvEntryValue.getValue()); |
|||
} |
|||
|
|||
@Override |
|||
public boolean updateEntry(ArgumentEntry entry) { |
|||
if (entry instanceof SingleValueArgumentEntry singleValueEntry) { |
|||
if (singleValueEntry.getTs() == this.ts) { |
|||
return false; |
|||
} |
|||
|
|||
Long newVersion = singleValueEntry.getVersion(); |
|||
if (newVersion == null || this.version == null || newVersion > this.version) { |
|||
this.ts = singleValueEntry.getTs(); |
|||
this.version = newVersion; |
|||
this.kvEntryValue = singleValueEntry.getKvEntryValue(); |
|||
return true; |
|||
} |
|||
} else { |
|||
throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); |
|||
} |
|||
return false; |
|||
} |
|||
} |
|||
@ -0,0 +1,146 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.cf.ctx.state; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnore; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.thingsboard.script.api.tbel.TbelCfArg; |
|||
import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; |
|||
import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.TreeMap; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
@AllArgsConstructor |
|||
@Slf4j |
|||
public class TsRollingArgumentEntry implements ArgumentEntry { |
|||
|
|||
private Integer limit; |
|||
private Long timeWindow; |
|||
private TreeMap<Long, Double> tsRecords = new TreeMap<>(); |
|||
|
|||
private boolean forceResetPrevious; |
|||
|
|||
public TsRollingArgumentEntry(List<TsKvEntry> kvEntries, int limit, long timeWindow) { |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); |
|||
} |
|||
|
|||
public TsRollingArgumentEntry(TreeMap<Long, Double> tsRecords, int limit, long timeWindow) { |
|||
this.tsRecords = tsRecords; |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
} |
|||
|
|||
public TsRollingArgumentEntry(int limit, long timeWindow) { |
|||
this.tsRecords = new TreeMap<>(); |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
} |
|||
|
|||
public TsRollingArgumentEntry(Integer limit, Long timeWindow, TreeMap<Long, Double> tsRecords) { |
|||
this.limit = limit; |
|||
this.timeWindow = timeWindow; |
|||
this.tsRecords = tsRecords; |
|||
} |
|||
|
|||
@Override |
|||
public ArgumentEntryType getType() { |
|||
return ArgumentEntryType.TS_ROLLING; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isEmpty() { |
|||
return tsRecords.isEmpty(); |
|||
} |
|||
|
|||
@JsonIgnore |
|||
@Override |
|||
public Object getValue() { |
|||
return tsRecords; |
|||
} |
|||
|
|||
@Override |
|||
public TbelCfArg toTbelCfArg() { |
|||
List<TbelCfTsDoubleVal> values = new ArrayList<>(tsRecords.size()); |
|||
for (var e : tsRecords.entrySet()) { |
|||
values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); |
|||
} |
|||
return new TbelCfTsRollingArg(timeWindow, values); |
|||
} |
|||
|
|||
@Override |
|||
public boolean updateEntry(ArgumentEntry entry) { |
|||
if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { |
|||
updateTsRollingEntry(tsRollingEntry); |
|||
} else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { |
|||
updateSingleValueEntry(singleValueEntry); |
|||
} else { |
|||
throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); |
|||
} |
|||
return true; |
|||
} |
|||
|
|||
private void updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { |
|||
for (Map.Entry<Long, Double> tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { |
|||
addTsRecord(tsRecordEntry.getKey(), tsRecordEntry.getValue()); |
|||
} |
|||
} |
|||
|
|||
private void updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { |
|||
addTsRecord(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); |
|||
} |
|||
|
|||
private void addTsRecord(Long ts, KvEntry value) { |
|||
try { |
|||
switch (value.getDataType()) { |
|||
case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); |
|||
case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); |
|||
case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); |
|||
case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); |
|||
case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); |
|||
} |
|||
} catch (Exception e) { |
|||
tsRecords.put(ts, Double.NaN); |
|||
log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); |
|||
} finally { |
|||
cleanupExpiredRecords(); |
|||
} |
|||
} |
|||
|
|||
private void addTsRecord(Long ts, double value) { |
|||
tsRecords.put(ts, value); |
|||
cleanupExpiredRecords(); |
|||
} |
|||
|
|||
private void cleanupExpiredRecords() { |
|||
if (tsRecords.size() > limit) { |
|||
tsRecords.pollFirstEntry(); |
|||
} |
|||
tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,82 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edge.rpc.processor.rule; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.data.util.Pair; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.id.RuleChainId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.rule.RuleChain; |
|||
import org.thingsboard.server.common.data.rule.RuleChainMetaData; |
|||
import org.thingsboard.server.common.data.rule.RuleChainType; |
|||
import org.thingsboard.server.common.data.rule.RuleNode; |
|||
import org.thingsboard.server.dao.service.DataValidator; |
|||
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; |
|||
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; |
|||
import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; |
|||
|
|||
import java.util.function.Function; |
|||
|
|||
@Slf4j |
|||
public class BaseRuleChainProcessor extends BaseEdgeProcessor { |
|||
|
|||
@Autowired |
|||
private DataValidator<RuleChain> ruleChainValidator; |
|||
|
|||
protected Pair<Boolean, Boolean> saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, RuleChainType ruleChainType) { |
|||
boolean created = false; |
|||
RuleChain ruleChainFromDb = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId); |
|||
if (ruleChainFromDb == null) { |
|||
created = true; |
|||
} |
|||
|
|||
RuleChain ruleChain = JacksonUtil.fromString(ruleChainUpdateMsg.getEntity(), RuleChain.class, true); |
|||
if (ruleChain == null) { |
|||
throw new RuntimeException("[{" + tenantId + "}] ruleChainUpdateMsg {" + ruleChainUpdateMsg + "} cannot be converted to rule chain"); |
|||
} |
|||
boolean isRoot = ruleChain.isRoot(); |
|||
if (RuleChainType.CORE.equals(ruleChainType)) { |
|||
ruleChain.setRoot(false); |
|||
} else { |
|||
ruleChain.setRoot(ruleChainFromDb == null ? false : ruleChainFromDb.isRoot()); |
|||
} |
|||
ruleChain.setType(ruleChainType); |
|||
|
|||
ruleChainValidator.validate(ruleChain, RuleChain::getTenantId); |
|||
if (created) { |
|||
ruleChain.setId(ruleChainId); |
|||
} |
|||
edgeCtx.getRuleChainService().saveRuleChain(ruleChain, true, false); |
|||
return Pair.of(created, isRoot); |
|||
} |
|||
|
|||
protected void saveOrUpdateRuleChainMetadata(TenantId tenantId, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) { |
|||
RuleChainMetaData ruleChainMetadata = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); |
|||
if (ruleChainMetadata == null) { |
|||
throw new RuntimeException("[{" + tenantId + "}] ruleChainMetadataUpdateMsg {" + ruleChainMetadataUpdateMsg + "} cannot be converted to rule chain metadata"); |
|||
} |
|||
if (!ruleChainMetadata.getNodes().isEmpty()) { |
|||
ruleChainMetadata.setVersion(null); |
|||
for (RuleNode ruleNode : ruleChainMetadata.getNodes()) { |
|||
ruleNode.setRuleChainId(null); |
|||
ruleNode.setId(null); |
|||
} |
|||
edgeCtx.getRuleChainService().saveRuleChainMetaData(tenantId, ruleChainMetadata, Function.identity(), true); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,117 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edqs; |
|||
|
|||
import com.google.common.util.concurrent.Futures; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.edqs.query.EdqsRequest; |
|||
import org.thingsboard.server.common.data.edqs.query.EdqsResponse; |
|||
import org.thingsboard.server.common.data.id.CustomerId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.msg.edqs.EdqsApiService; |
|||
import org.thingsboard.server.edqs.state.EdqsPartitionService; |
|||
import org.thingsboard.server.gen.transport.TransportProtos; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; |
|||
import org.thingsboard.server.queue.TbQueueRequestTemplate; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')") |
|||
public class DefaultEdqsApiService implements EdqsApiService { |
|||
|
|||
private final EdqsPartitionService edqsPartitionService; |
|||
private final EdqsClientQueueFactory queueFactory; |
|||
private TbQueueRequestTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> requestTemplate; |
|||
|
|||
@Value("${queue.edqs.api.auto_enable:true}") |
|||
private boolean autoEnable; |
|||
|
|||
private Boolean apiEnabled = null; |
|||
|
|||
@PostConstruct |
|||
private void init() { |
|||
requestTemplate = queueFactory.createEdqsRequestTemplate(); |
|||
requestTemplate.init(); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<EdqsResponse> processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { |
|||
var requestMsg = ToEdqsMsg.newBuilder() |
|||
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) |
|||
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) |
|||
.setTs(System.currentTimeMillis()) |
|||
.setRequestMsg(TransportProtos.EdqsRequestMsg.newBuilder() |
|||
.setValue(JacksonUtil.toString(request)) |
|||
.build()); |
|||
if (customerId != null && !customerId.isNullUid()) { |
|||
requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits()); |
|||
requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits()); |
|||
} |
|||
|
|||
Integer partition = edqsPartitionService.resolvePartition(tenantId); |
|||
ListenableFuture<TbProtoQueueMsg<FromEdqsMsg>> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition); |
|||
return Futures.transform(resultFuture, msg -> { |
|||
TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg(); |
|||
return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class); |
|||
}, MoreExecutors.directExecutor()); |
|||
} |
|||
|
|||
@Override |
|||
public boolean isEnabled() { |
|||
return Boolean.TRUE.equals(apiEnabled); |
|||
} |
|||
|
|||
@Override |
|||
public void setEnabled(boolean enabled) { |
|||
if (enabled) { |
|||
log.info("Enabling EDQS API"); |
|||
} else { |
|||
log.info("Disabling EDQS API"); |
|||
} |
|||
apiEnabled = enabled; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isSupported() { |
|||
return true; |
|||
} |
|||
|
|||
@Override |
|||
public boolean isAutoEnable() { |
|||
return autoEnable; |
|||
} |
|||
|
|||
@PreDestroy |
|||
private void stop() { |
|||
requestTemplate.stop(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,298 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edqs; |
|||
|
|||
import com.google.protobuf.ByteString; |
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.SneakyThrows; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
import org.springframework.context.annotation.Lazy; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.common.util.ThingsBoardExecutors; |
|||
import org.thingsboard.server.cluster.TbClusterService; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.edqs.EdqsEventType; |
|||
import org.thingsboard.server.common.data.edqs.EdqsObject; |
|||
import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; |
|||
import org.thingsboard.server.common.data.edqs.Entity; |
|||
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; |
|||
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.JsonDataEntry; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.msg.edqs.EdqsApiService; |
|||
import org.thingsboard.server.common.msg.edqs.EdqsService; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.edqs.processor.EdqsProducer; |
|||
import org.thingsboard.server.edqs.state.EdqsPartitionService; |
|||
import org.thingsboard.server.edqs.util.EdqsConverter; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; |
|||
import org.thingsboard.server.queue.discovery.HashPartitionService; |
|||
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; |
|||
import org.thingsboard.server.queue.discovery.TopicService; |
|||
import org.thingsboard.server.queue.edqs.EdqsQueue; |
|||
import org.thingsboard.server.queue.environment.DistributedLock; |
|||
import org.thingsboard.server.queue.environment.DistributedLockService; |
|||
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
|
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") |
|||
public class DefaultEdqsService implements EdqsService { |
|||
|
|||
private final EdqsClientQueueFactory queueFactory; |
|||
private final EdqsConverter edqsConverter; |
|||
private final EdqsSyncService edqsSyncService; |
|||
private final EdqsApiService edqsApiService; |
|||
private final DistributedLockService distributedLockService; |
|||
private final AttributesService attributesService; |
|||
private final EdqsPartitionService edqsPartitionService; |
|||
private final TopicService topicService; |
|||
private final TbServiceInfoProvider serviceInfoProvider; |
|||
@Autowired @Lazy |
|||
private TbClusterService clusterService; |
|||
@Autowired @Lazy |
|||
private HashPartitionService hashPartitionService; |
|||
|
|||
private EdqsProducer eventsProducer; |
|||
private ExecutorService executor; |
|||
private DistributedLock syncLock; |
|||
|
|||
@PostConstruct |
|||
private void init() { |
|||
executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); |
|||
eventsProducer = EdqsProducer.builder() |
|||
.queue(EdqsQueue.EVENTS) |
|||
.partitionService(edqsPartitionService) |
|||
.topicService(topicService) |
|||
.producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) |
|||
.build(); |
|||
syncLock = distributedLockService.getLock("edqs_sync"); |
|||
} |
|||
|
|||
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) |
|||
public void onStartUp() { |
|||
if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) { |
|||
return; |
|||
} |
|||
executor.submit(() -> { |
|||
try { |
|||
EdqsSyncState syncState = getSyncState(); |
|||
if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { |
|||
if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { |
|||
processSystemRequest(ToCoreEdqsRequest.builder() |
|||
.syncRequest(new EdqsSyncRequest()) |
|||
.build()); |
|||
} |
|||
} else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) { |
|||
// only if topic/RocksDB is not empty and sync is finished
|
|||
edqsApiService.setEnabled(true); |
|||
} |
|||
} catch (Throwable e) { |
|||
log.error("Failed to start EDQS service", e); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
@Override |
|||
public void processSystemRequest(ToCoreEdqsRequest request) { |
|||
log.info("Processing system request {}", request); |
|||
if (request.getSyncRequest() != null) { |
|||
saveSyncState(EdqsSyncStatus.REQUESTED); |
|||
} |
|||
broadcast(request.toInternalMsg()); |
|||
} |
|||
|
|||
@Override |
|||
public void processSystemMsg(ToCoreEdqsMsg msg) { |
|||
executor.submit(() -> { |
|||
log.info("Processing system msg {}", msg); |
|||
try { |
|||
if (msg.getApiEnabled() != null) { |
|||
edqsApiService.setEnabled(msg.getApiEnabled()); |
|||
} |
|||
|
|||
if (msg.getSyncRequest() != null) { |
|||
syncLock.lock(); |
|||
try { |
|||
EdqsSyncState syncState = getSyncState(); |
|||
if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) { |
|||
log.info("EDQS sync is already finished"); |
|||
return; |
|||
} |
|||
|
|||
saveSyncState(EdqsSyncStatus.STARTED); |
|||
edqsSyncService.sync(); |
|||
saveSyncState(EdqsSyncStatus.FINISHED); |
|||
|
|||
if (edqsApiService.isSupported()) |
|||
if (edqsApiService.isAutoEnable()) { |
|||
log.info("EDQS sync is finished, auto-enabling API"); |
|||
broadcast(ToCoreEdqsMsg.builder() |
|||
.apiEnabled(Boolean.TRUE) |
|||
.build()); |
|||
} else { |
|||
log.info("EDQS sync is finished, but leaving API disabled"); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Failed to complete sync", e); |
|||
saveSyncState(EdqsSyncStatus.FAILED); |
|||
} finally { |
|||
syncLock.unlock(); |
|||
} |
|||
} |
|||
} catch (Throwable e) { |
|||
log.error("Failed to process msg {}", msg, e); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
@Override |
|||
public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) { |
|||
EntityType entityType = entityId.getEntityType(); |
|||
ObjectType objectType = ObjectType.fromEntityType(entityType); |
|||
if (!isEdqsType(tenantId, objectType)) { |
|||
log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType); |
|||
return; |
|||
} |
|||
onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity)); |
|||
} |
|||
|
|||
@Override |
|||
public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) { |
|||
processEvent(tenantId, objectType, EdqsEventType.UPDATED, object); |
|||
} |
|||
|
|||
@Override |
|||
public void onDelete(TenantId tenantId, EntityId entityId) { |
|||
EntityType entityType = entityId.getEntityType(); |
|||
ObjectType objectType = ObjectType.fromEntityType(entityType); |
|||
if (!isEdqsType(tenantId, objectType)) { |
|||
log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType); |
|||
return; |
|||
} |
|||
onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE)); |
|||
} |
|||
|
|||
@Override |
|||
public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) { |
|||
processEvent(tenantId, objectType, EdqsEventType.DELETED, object); |
|||
} |
|||
|
|||
protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) { |
|||
executor.submit(() -> { |
|||
try { |
|||
String key = object.key(); |
|||
Long version = object.version(); |
|||
EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder() |
|||
.setKey(key) |
|||
.setObjectType(objectType.name()) |
|||
.setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object))) |
|||
.setEventType(eventType.name()); |
|||
if (version != null) { |
|||
eventMsg.setVersion(version); |
|||
} |
|||
eventsProducer.send(tenantId, objectType, key, ToEdqsMsg.newBuilder() |
|||
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) |
|||
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) |
|||
.setTs(System.currentTimeMillis()) |
|||
.setEventMsg(eventMsg) |
|||
.build()); |
|||
} catch (Throwable e) { |
|||
log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private boolean isEdqsType(TenantId tenantId, ObjectType objectType) { |
|||
if (objectType == null) { |
|||
return false; |
|||
} |
|||
if (!tenantId.isSysTenantId()) { |
|||
return ObjectType.edqsTypes.contains(objectType); |
|||
} else { |
|||
return ObjectType.edqsSystemTypes.contains(objectType); |
|||
} |
|||
} |
|||
|
|||
private void broadcast(ToCoreEdqsMsg msg) { |
|||
clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder() |
|||
.setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder() |
|||
.setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg)))) |
|||
.build()); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private EdqsSyncState getSyncState() { |
|||
EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS) |
|||
.flatMap(KvEntry::getJsonValue) |
|||
.map(value -> JacksonUtil.fromString(value, EdqsSyncState.class)) |
|||
.orElse(null); |
|||
log.info("EDQS sync state: {}", state); |
|||
return state; |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private void saveSyncState(EdqsSyncStatus status) { |
|||
EdqsSyncState state = new EdqsSyncState(status); |
|||
log.info("New EDQS sync state: {}", state); |
|||
attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry( |
|||
new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)), |
|||
System.currentTimeMillis())).get(30, TimeUnit.SECONDS); |
|||
} |
|||
|
|||
@PreDestroy |
|||
private void stop() { |
|||
executor.shutdown(); |
|||
eventsProducer.stop(); |
|||
} |
|||
|
|||
@Data |
|||
@AllArgsConstructor |
|||
@NoArgsConstructor |
|||
private static class EdqsSyncState { |
|||
private EdqsSyncStatus status; |
|||
} |
|||
|
|||
private enum EdqsSyncStatus { |
|||
REQUESTED, |
|||
STARTED, |
|||
FINISHED, |
|||
FAILED |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,61 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edqs; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.transaction.event.TransactionalEventListener; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.audit.ActionType; |
|||
import org.thingsboard.server.common.msg.edqs.EdqsService; |
|||
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; |
|||
import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; |
|||
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") |
|||
public class EdqsListener { |
|||
|
|||
private final EdqsService edqsService; |
|||
|
|||
@TransactionalEventListener(fallbackExecution = true) |
|||
public void onUpdate(SaveEntityEvent<?> event) { |
|||
if (event.getEntityId() == null || event.getEntity() == null) { |
|||
return; |
|||
} |
|||
edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity()); |
|||
} |
|||
|
|||
@TransactionalEventListener(fallbackExecution = true) |
|||
public void onDelete(DeleteEntityEvent<?> event) { |
|||
if (event.getEntityId() == null) { |
|||
return; |
|||
} |
|||
edqsService.onDelete(event.getTenantId(), event.getEntityId()); |
|||
} |
|||
|
|||
@TransactionalEventListener(fallbackExecution = true) |
|||
public void handleEvent(RelationActionEvent relationEvent) { |
|||
if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { |
|||
edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); |
|||
} else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { |
|||
edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,284 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edqs; |
|||
|
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.annotation.Lazy; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.edqs.AttributeKv; |
|||
import org.thingsboard.server.common.data.edqs.EdqsEventType; |
|||
import org.thingsboard.server.common.data.edqs.EdqsObject; |
|||
import org.thingsboard.server.common.data.edqs.Entity; |
|||
import org.thingsboard.server.common.data.edqs.LatestTsKv; |
|||
import org.thingsboard.server.common.data.edqs.fields.EntityFields; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageDataIterable; |
|||
import org.thingsboard.server.common.data.relation.RelationTypeGroup; |
|||
import org.thingsboard.server.dao.Dao; |
|||
import org.thingsboard.server.dao.attributes.AttributesDao; |
|||
import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; |
|||
import org.thingsboard.server.dao.entity.EntityDaoRegistry; |
|||
import org.thingsboard.server.dao.model.sql.AttributeKvEntity; |
|||
import org.thingsboard.server.dao.model.sql.RelationEntity; |
|||
import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; |
|||
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; |
|||
import org.thingsboard.server.dao.sql.relation.RelationRepository; |
|||
import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; |
|||
|
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; |
|||
import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; |
|||
import static org.thingsboard.server.common.data.ObjectType.RELATION; |
|||
import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; |
|||
|
|||
@Slf4j |
|||
public abstract class EdqsSyncService { |
|||
|
|||
@Value("${queue.edqs.sync.entity_batch_size:10000}") |
|||
private int entityBatchSize; |
|||
@Value("${queue.edqs.sync.ts_batch_size:10000}") |
|||
private int tsBatchSize; |
|||
@Autowired |
|||
private EntityDaoRegistry entityDaoRegistry; |
|||
@Autowired |
|||
private AttributesDao attributesDao; |
|||
@Autowired |
|||
private KeyDictionaryDao keyDictionaryDao; |
|||
@Autowired |
|||
private RelationRepository relationRepository; |
|||
@Autowired |
|||
private TsKvLatestRepository tsKvLatestRepository; |
|||
@Autowired |
|||
@Lazy |
|||
private DefaultEdqsService edqsService; |
|||
|
|||
private final ConcurrentHashMap<UUID, EntityIdInfo> entityInfoMap = new ConcurrentHashMap<>(); |
|||
private final ConcurrentHashMap<Integer, String> keys = new ConcurrentHashMap<>(); |
|||
|
|||
private final Map<ObjectType, AtomicInteger> counters = new ConcurrentHashMap<>(); |
|||
|
|||
public abstract boolean isSyncNeeded(); |
|||
|
|||
public void sync() { |
|||
log.info("Synchronizing data to EDQS"); |
|||
long startTs = System.currentTimeMillis(); |
|||
counters.clear(); |
|||
|
|||
syncTenantEntities(); |
|||
syncRelations(); |
|||
loadKeyDictionary(); |
|||
syncAttributes(); |
|||
syncLatestTimeseries(); |
|||
|
|||
counters.clear(); |
|||
log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); |
|||
} |
|||
|
|||
private void process(TenantId tenantId, ObjectType type, EdqsObject object) { |
|||
AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger()); |
|||
if (counter.incrementAndGet() % 10000 == 0) { |
|||
log.info("Processed {} {} objects", counter.get(), type); |
|||
} |
|||
edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); |
|||
} |
|||
|
|||
private void syncTenantEntities() { |
|||
for (ObjectType type : edqsTenantTypes) { |
|||
log.info("Synchronizing {} entities to EDQS", type); |
|||
long ts = System.currentTimeMillis(); |
|||
EntityType entityType = type.toEntityType(); |
|||
Dao<?> dao = entityDaoRegistry.getDao(entityType); |
|||
UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
|||
while (true) { |
|||
var batch = dao.findNextBatch(lastId, entityBatchSize); |
|||
if (batch.isEmpty()) { |
|||
break; |
|||
} |
|||
for (EntityFields entityFields : batch) { |
|||
TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); |
|||
entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); |
|||
process(tenantId, type, new Entity(entityType, entityFields)); |
|||
} |
|||
EntityFields lastRecord = batch.get(batch.size() - 1); |
|||
lastId = lastRecord.getId(); |
|||
} |
|||
log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); |
|||
} |
|||
} |
|||
|
|||
private void syncRelations() { |
|||
log.info("Synchronizing relations to EDQS"); |
|||
long ts = System.currentTimeMillis(); |
|||
UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
|||
String lastFromEntityType = ""; |
|||
String lastRelationTypeGroup = ""; |
|||
String lastRelationType = ""; |
|||
UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
|||
String lastToEntityType = ""; |
|||
|
|||
while (true) { |
|||
List<RelationEntity> batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, |
|||
lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize); |
|||
if (batch.isEmpty()) { |
|||
break; |
|||
} |
|||
processRelationBatch(batch); |
|||
|
|||
RelationEntity lastRecord = batch.get(batch.size() - 1); |
|||
lastFromEntityId = lastRecord.getFromId(); |
|||
lastFromEntityType = lastRecord.getFromType(); |
|||
lastRelationTypeGroup = lastRecord.getRelationTypeGroup(); |
|||
lastRelationType = lastRecord.getRelationType(); |
|||
lastToEntityId = lastRecord.getToId(); |
|||
lastToEntityType = lastRecord.getToType(); |
|||
} |
|||
log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); |
|||
} |
|||
|
|||
private void processRelationBatch(List<RelationEntity> relations) { |
|||
for (RelationEntity relation : relations) { |
|||
if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) { |
|||
EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); |
|||
if (entityIdInfo != null) { |
|||
process(entityIdInfo.tenantId(), RELATION, relation.toData()); |
|||
} else { |
|||
log.info("Relation from id not found: {} ", relation); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void loadKeyDictionary() { |
|||
log.info("Loading key dictionary"); |
|||
long ts = System.currentTimeMillis(); |
|||
var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000); |
|||
for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) { |
|||
keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey()); |
|||
} |
|||
log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts)); |
|||
} |
|||
|
|||
private void syncAttributes() { |
|||
log.info("Synchronizing attributes to EDQS"); |
|||
long ts = System.currentTimeMillis(); |
|||
|
|||
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
|||
int lastAttributeType = Integer.MIN_VALUE; |
|||
int lastAttributeKey = Integer.MIN_VALUE; |
|||
|
|||
while (true) { |
|||
List<AttributeKvEntity> batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize); |
|||
if (batch.isEmpty()) { |
|||
break; |
|||
} |
|||
processAttributeBatch(batch); |
|||
|
|||
AttributeKvEntity lastRecord = batch.get(batch.size() - 1); |
|||
lastEntityId = lastRecord.getId().getEntityId(); |
|||
lastAttributeType = lastRecord.getId().getAttributeType(); |
|||
lastAttributeKey = lastRecord.getId().getAttributeKey(); |
|||
} |
|||
log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); |
|||
} |
|||
|
|||
private void processAttributeBatch(List<AttributeKvEntity> batch) { |
|||
for (AttributeKvEntity attribute : batch) { |
|||
attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); |
|||
UUID entityId = attribute.getId().getEntityId(); |
|||
EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); |
|||
if (entityIdInfo == null) { |
|||
log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId); |
|||
continue; |
|||
} |
|||
AttributeKv attributeKv = new AttributeKv( |
|||
EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId), |
|||
AttributeScope.valueOf(attribute.getId().getAttributeType()), |
|||
attribute.toData(), |
|||
attribute.getVersion()); |
|||
process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); |
|||
} |
|||
} |
|||
|
|||
private void syncLatestTimeseries() { |
|||
log.info("Synchronizing latest timeseries to EDQS"); |
|||
long ts = System.currentTimeMillis(); |
|||
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
|||
int lastKey = Integer.MIN_VALUE; |
|||
|
|||
while (true) { |
|||
List<TsKvLatestEntity> batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize); |
|||
if (batch.isEmpty()) { |
|||
break; |
|||
} |
|||
processTsKvLatestBatch(batch); |
|||
|
|||
TsKvLatestEntity lastRecord = batch.get(batch.size() - 1); |
|||
lastEntityId = lastRecord.getEntityId(); |
|||
lastKey = lastRecord.getKey(); |
|||
} |
|||
log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); |
|||
} |
|||
|
|||
private void processTsKvLatestBatch(List<TsKvLatestEntity> tsKvLatestEntities) { |
|||
for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { |
|||
try { |
|||
String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); |
|||
if (strKey == null) { |
|||
log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey()); |
|||
continue; |
|||
} |
|||
tsKvLatestEntity.setStrKey(strKey); |
|||
UUID entityUuid = tsKvLatestEntity.getEntityId(); |
|||
EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid); |
|||
if (entityIdInfo != null) { |
|||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid); |
|||
LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion()); |
|||
process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private String getStrKeyOrFetchFromDb(int key) { |
|||
String strKey = keys.get(key); |
|||
if (strKey != null) { |
|||
return strKey; |
|||
} else { |
|||
strKey = keyDictionaryDao.getKey(key); |
|||
if (strKey != null) { |
|||
keys.put(key, strKey); |
|||
} |
|||
} |
|||
return strKey; |
|||
} |
|||
|
|||
public record EntityIdInfo(EntityType entityType, TenantId tenantId) { |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,42 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edqs; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.queue.edqs.EdqsQueue; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaAdmin; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaSettings; |
|||
|
|||
import java.util.Collections; |
|||
|
|||
@Service |
|||
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") |
|||
public class KafkaEdqsSyncService extends EdqsSyncService { |
|||
|
|||
private final boolean syncNeeded; |
|||
|
|||
public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) { |
|||
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); |
|||
this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); |
|||
} |
|||
|
|||
@Override |
|||
public boolean isSyncNeeded() { |
|||
return syncNeeded; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.edqs; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.edqs.util.EdqsRocksDb; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") |
|||
public class LocalEdqsSyncService extends EdqsSyncService { |
|||
|
|||
private final EdqsRocksDb db; |
|||
|
|||
@Override |
|||
public boolean isSyncNeeded() { |
|||
return db.isNew(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,106 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.entitiy.cf; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.transaction.annotation.Transactional; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.audit.ActionType; |
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.entitiy.AbstractTbEntityService; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
|
|||
import java.util.Optional; |
|||
|
|||
@TbCoreComponent |
|||
@Service |
|||
@Slf4j |
|||
@RequiredArgsConstructor |
|||
public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { |
|||
|
|||
private final CalculatedFieldService calculatedFieldService; |
|||
|
|||
@Override |
|||
public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { |
|||
ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; |
|||
TenantId tenantId = calculatedField.getTenantId(); |
|||
try { |
|||
if (ActionType.UPDATED.equals(actionType)) { |
|||
CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); |
|||
checkForEntityChange(existingCf, calculatedField); |
|||
} |
|||
checkEntityExistence(tenantId, calculatedField.getEntityId()); |
|||
CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); |
|||
logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); |
|||
return savedCalculatedField; |
|||
} catch (ThingsboardException e) { |
|||
logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { |
|||
return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); |
|||
} |
|||
|
|||
@Override |
|||
public PageData<CalculatedField> findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { |
|||
TenantId tenantId = user.getTenantId(); |
|||
checkEntityExistence(tenantId, entityId); |
|||
return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); |
|||
} |
|||
|
|||
@Override |
|||
@Transactional |
|||
public void delete(CalculatedField calculatedField, SecurityUser user) { |
|||
ActionType actionType = ActionType.DELETED; |
|||
TenantId tenantId = calculatedField.getTenantId(); |
|||
CalculatedFieldId calculatedFieldId = calculatedField.getId(); |
|||
try { |
|||
calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); |
|||
logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); |
|||
} catch (Exception e) { |
|||
logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); |
|||
throw e; |
|||
} |
|||
} |
|||
|
|||
private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { |
|||
if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { |
|||
throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); |
|||
} |
|||
} |
|||
|
|||
private void checkEntityExistence(TenantId tenantId, EntityId entityId) { |
|||
switch (entityId.getEntityType()) { |
|||
case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) |
|||
.orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); |
|||
default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.entitiy.cf; |
|||
|
|||
import org.thingsboard.server.common.data.cf.CalculatedField; |
|||
import org.thingsboard.server.common.data.exception.ThingsboardException; |
|||
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
|
|||
public interface TbCalculatedFieldService { |
|||
|
|||
CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; |
|||
|
|||
CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); |
|||
|
|||
PageData<CalculatedField> findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); |
|||
|
|||
void delete(CalculatedField calculatedField, SecurityUser user); |
|||
|
|||
} |
|||
@ -0,0 +1,43 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.housekeeper.processor; |
|||
|
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; |
|||
import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; |
|||
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
|||
|
|||
@Component |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor<HousekeeperTask> { |
|||
|
|||
private final CalculatedFieldService calculatedFieldService; |
|||
|
|||
@Override |
|||
public void process(HousekeeperTask task) throws Exception { |
|||
int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); |
|||
log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); |
|||
} |
|||
|
|||
@Override |
|||
public HousekeeperTaskType getTaskType() { |
|||
return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,269 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.queue; |
|||
|
|||
import com.google.common.util.concurrent.ListeningExecutorService; |
|||
import com.google.common.util.concurrent.MoreExecutors; |
|||
import jakarta.annotation.PostConstruct; |
|||
import jakarta.annotation.PreDestroy; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.context.ApplicationEventPublisher; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.ThingsBoardExecutors; |
|||
import org.thingsboard.server.actors.ActorSystemContext; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; |
|||
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.queue.QueueConfig; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
|||
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
|||
import org.thingsboard.server.common.msg.queue.ServiceType; |
|||
import org.thingsboard.server.common.msg.queue.TbCallback; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.tenant.TbTenantProfileCache; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; |
|||
import org.thingsboard.server.queue.TbQueueConsumer; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
import org.thingsboard.server.queue.discovery.QueueKey; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; |
|||
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
|||
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldCache; |
|||
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
|||
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
|||
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
|||
import org.thingsboard.server.service.queue.processing.AbstractConsumerService; |
|||
import org.thingsboard.server.service.queue.processing.IdMsgPair; |
|||
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; |
|||
|
|||
import java.util.List; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.concurrent.ConcurrentMap; |
|||
import java.util.concurrent.CountDownLatch; |
|||
import java.util.concurrent.Future; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.stream.Collectors; |
|||
|
|||
import static org.thingsboard.server.common.util.ProtoUtils.fromProto; |
|||
|
|||
@Service |
|||
@TbRuleEngineComponent |
|||
@Slf4j |
|||
public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService<ToCalculatedFieldNotificationMsg> implements TbCalculatedFieldConsumerService { |
|||
|
|||
@Value("${queue.calculated_fields.poll_interval:25}") |
|||
private long pollInterval; |
|||
@Value("${queue.calculated_fields.pack_processing_timeout:60000}") |
|||
private long packProcessingTimeout; |
|||
@Value("${queue.calculated_fields.pool_size:8}") |
|||
private int poolSize; |
|||
|
|||
private final TbRuleEngineQueueFactory queueFactory; |
|||
private final CalculatedFieldStateService stateService; |
|||
|
|||
private PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer; |
|||
|
|||
public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, |
|||
ActorSystemContext actorContext, |
|||
TbDeviceProfileCache deviceProfileCache, |
|||
TbAssetProfileCache assetProfileCache, |
|||
TbTenantProfileCache tenantProfileCache, |
|||
TbApiUsageStateService apiUsageStateService, |
|||
PartitionService partitionService, |
|||
ApplicationEventPublisher eventPublisher, |
|||
JwtSettingsService jwtSettingsService, |
|||
CalculatedFieldCache calculatedFieldCache, |
|||
CalculatedFieldStateService stateService) { |
|||
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, |
|||
eventPublisher, jwtSettingsService); |
|||
this.queueFactory = tbQueueFactory; |
|||
this.stateService = stateService; |
|||
} |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
super.init("tb-cf"); |
|||
|
|||
this.eventConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToCalculatedFieldMsg>>create() |
|||
.queueKey(QueueKey.CF) |
|||
.topic(partitionService.getTopic(QueueKey.CF)) |
|||
.pollInterval(pollInterval) |
|||
.msgPackProcessor(this::processMsgs) |
|||
.consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) |
|||
.consumerExecutor(consumersExecutor) |
|||
.scheduler(scheduler) |
|||
.taskExecutor(mgmtExecutor) |
|||
.build(); |
|||
stateService.init(eventConsumer); |
|||
} |
|||
|
|||
@PreDestroy |
|||
public void destroy() { |
|||
super.destroy(); |
|||
} |
|||
|
|||
@Override |
|||
protected void startConsumers() { |
|||
super.startConsumers(); |
|||
} |
|||
|
|||
@Override |
|||
protected void onTbApplicationEvent(PartitionChangeEvent event) { |
|||
var partitions = event.getCfPartitions(); |
|||
try { |
|||
stateService.restore(partitions); |
|||
// eventConsumer's partitions will be updated by stateService
|
|||
|
|||
// Cleanup old entities after corresponding consumers are stopped.
|
|||
// Any periodic tasks need to check that the entity is still managed by the current server before processing.
|
|||
actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); |
|||
} catch (Throwable t) { |
|||
log.error("Failed to process partition change event: {}", event, t); |
|||
} |
|||
} |
|||
|
|||
private boolean[] partitionsToBooleanIndexArray(Set<TopicPartitionInfo> partitions) { |
|||
boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; |
|||
for (var tpi : partitions) { |
|||
tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); |
|||
} |
|||
return myPartitions; |
|||
} |
|||
|
|||
private void processMsgs(List<TbProtoQueueMsg<ToCalculatedFieldMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldMsg>> consumer, QueueConfig config) throws Exception { |
|||
List<IdMsgPair<ToCalculatedFieldMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); |
|||
ConcurrentMap<UUID, TbProtoQueueMsg<ToCalculatedFieldMsg>> pendingMap = orderedMsgList.stream().collect( |
|||
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); |
|||
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); |
|||
TbPackProcessingContext<TbProtoQueueMsg<ToCalculatedFieldMsg>> ctx = new TbPackProcessingContext<>( |
|||
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); |
|||
PendingMsgHolder<ToCalculatedFieldMsg> pendingMsgHolder = new PendingMsgHolder<>(); |
|||
Future<?> packSubmitFuture = consumersExecutor.submit(() -> { |
|||
orderedMsgList.forEach((element) -> { |
|||
UUID id = element.getUuid(); |
|||
TbProtoQueueMsg<ToCalculatedFieldMsg> msg = element.getMsg(); |
|||
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); |
|||
TbCallback callback = new TbPackCallback<>(id, ctx); |
|||
try { |
|||
ToCalculatedFieldMsg toCfMsg = msg.getValue(); |
|||
pendingMsgHolder.setMsg(toCfMsg); |
|||
if (toCfMsg.hasTelemetryMsg()) { |
|||
log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); |
|||
forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); |
|||
} else if (toCfMsg.hasLinkedTelemetryMsg()) { |
|||
forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); |
|||
} else if (toCfMsg.hasComponentLifecycleMsg()) { |
|||
log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); |
|||
forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); |
|||
} |
|||
} catch (Throwable e) { |
|||
log.warn("[{}] Failed to process message: {}", id, msg, e); |
|||
callback.onFailure(e); |
|||
} |
|||
}); |
|||
}); |
|||
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { |
|||
if (!packSubmitFuture.isDone()) { |
|||
packSubmitFuture.cancel(true); |
|||
log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); |
|||
} |
|||
if (log.isDebugEnabled()) { |
|||
ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); |
|||
} |
|||
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); |
|||
} |
|||
consumer.commit(); |
|||
} |
|||
|
|||
@Override |
|||
protected ServiceType getServiceType() { |
|||
return ServiceType.TB_RULE_ENGINE; |
|||
} |
|||
|
|||
@Override |
|||
protected long getNotificationPollDuration() { |
|||
return pollInterval; |
|||
} |
|||
|
|||
@Override |
|||
protected long getNotificationPackProcessingTimeout() { |
|||
return packProcessingTimeout; |
|||
} |
|||
|
|||
@Override |
|||
protected int getMgmtThreadPoolSize() { |
|||
return Math.max(Runtime.getRuntime().availableProcessors(), 4); |
|||
} |
|||
|
|||
@Override |
|||
protected TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldNotificationMsg>> createNotificationsConsumer() { |
|||
return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); |
|||
} |
|||
|
|||
@Override |
|||
protected void handleNotification(UUID id, TbProtoQueueMsg<ToCalculatedFieldNotificationMsg> msg, TbCallback callback) { |
|||
ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); |
|||
if (toCfNotification.hasComponentLifecycleMsg()) { |
|||
// from upstream (maybe removed since we don't need to init state for each partition)
|
|||
log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); |
|||
forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); |
|||
} else if (toCfNotification.hasLinkedTelemetryMsg()) { |
|||
forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); |
|||
} |
|||
} |
|||
|
|||
private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { |
|||
var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); |
|||
var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); |
|||
actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); |
|||
} |
|||
|
|||
private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { |
|||
var msg = linkedMsg.getMsg(); |
|||
var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); |
|||
var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); |
|||
actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); |
|||
} |
|||
|
|||
private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { |
|||
var msg = fromProto(proto); |
|||
actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); |
|||
} |
|||
|
|||
private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { |
|||
return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); |
|||
} |
|||
|
|||
@Override |
|||
protected void stopConsumers() { |
|||
super.stopConsumers(); |
|||
eventConsumer.stop(); |
|||
eventConsumer.awaitStop(); |
|||
stateService.stop(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.queue; |
|||
|
|||
import lombok.Getter; |
|||
import lombok.Setter; |
|||
|
|||
public class PendingMsgHolder<T> { |
|||
@Getter @Setter |
|||
private volatile T msg; |
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.queue; |
|||
|
|||
import org.springframework.context.ApplicationListener; |
|||
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
|||
|
|||
public interface TbCalculatedFieldConsumerService extends ApplicationListener<PartitionChangeEvent> { |
|||
|
|||
} |
|||
@ -1,48 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2025 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.queue.ruleengine; |
|||
|
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Getter; |
|||
import lombok.ToString; |
|||
import org.thingsboard.server.common.data.queue.QueueConfig; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
|
|||
import java.util.Set; |
|||
|
|||
@Getter |
|||
@ToString |
|||
@AllArgsConstructor |
|||
public class TbQueueConsumerManagerTask { |
|||
|
|||
private final QueueEvent event; |
|||
private QueueConfig config; |
|||
private Set<TopicPartitionInfo> partitions; |
|||
private boolean drainQueue; |
|||
|
|||
public static TbQueueConsumerManagerTask delete(boolean drainQueue) { |
|||
return new TbQueueConsumerManagerTask(QueueEvent.DELETE, null, null, drainQueue); |
|||
} |
|||
|
|||
public static TbQueueConsumerManagerTask configUpdate(QueueConfig config) { |
|||
return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, config, null, false); |
|||
} |
|||
|
|||
public static TbQueueConsumerManagerTask partitionChange(Set<TopicPartitionInfo> partitions) { |
|||
return new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, null, partitions, false); |
|||
} |
|||
|
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue