795 changed files with 39537 additions and 2808 deletions
@ -0,0 +1,70 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.common.util.DebugModeUtil; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.service.ContextAwareActor; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.TbActorMsg; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
|
||||
|
@Slf4j |
||||
|
public abstract class AbstractCalculatedFieldActor extends ContextAwareActor { |
||||
|
|
||||
|
protected final TenantId tenantId; |
||||
|
|
||||
|
public AbstractCalculatedFieldActor(ActorSystemContext systemContext, TenantId tenantId) { |
||||
|
super(systemContext); |
||||
|
this.tenantId = tenantId; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected boolean doProcess(TbActorMsg msg) { |
||||
|
if (msg instanceof ToCalculatedFieldSystemMsg cfm) { |
||||
|
Exception cause; |
||||
|
try { |
||||
|
return doProcessCfMsg(cfm); |
||||
|
} catch (CalculatedFieldException cfe) { |
||||
|
if (DebugModeUtil.isDebugFailuresAvailable(cfe.getCtx().getCalculatedField())) { |
||||
|
String message; |
||||
|
if (cfe.getErrorMessage() != null) { |
||||
|
message = cfe.getErrorMessage(); |
||||
|
} else if (cfe.getCause() != null) { |
||||
|
message = cfe.getCause().getMessage(); |
||||
|
} else { |
||||
|
message = "N/A"; |
||||
|
} |
||||
|
systemContext.persistCalculatedFieldDebugEvent(tenantId, cfe.getCtx().getCfId(), cfe.getEventEntity(), cfe.getArguments(), cfe.getMsgId(), cfe.getMsgType(), null, message); |
||||
|
} |
||||
|
cause = cfe.getCause(); |
||||
|
} catch (Exception e) { |
||||
|
logProcessingException(e); |
||||
|
cause = e; |
||||
|
} |
||||
|
cfm.getCallback().onFailure(cause); |
||||
|
return true; |
||||
|
} else { |
||||
|
return false; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
abstract void logProcessingException(Exception e); |
||||
|
|
||||
|
abstract boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException; |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,81 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.TbActorCtx; |
||||
|
import org.thingsboard.server.actors.TbActorException; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
||||
|
|
||||
|
@Slf4j |
||||
|
public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { |
||||
|
|
||||
|
private final CalculatedFieldEntityMessageProcessor processor; |
||||
|
|
||||
|
CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { |
||||
|
super(systemContext, tenantId); |
||||
|
this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void init(TbActorCtx ctx) throws TbActorException { |
||||
|
super.init(ctx); |
||||
|
log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); |
||||
|
try { |
||||
|
processor.init(ctx); |
||||
|
log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); |
||||
|
} catch (Exception e) { |
||||
|
log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); |
||||
|
throw new TbActorException("Failed to initialize CF entity actor", e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { |
||||
|
switch (msg.getMsgType()) { |
||||
|
case CF_PARTITIONS_CHANGE_MSG: |
||||
|
processor.process((CalculatedFieldPartitionChangeMsg) msg); |
||||
|
break; |
||||
|
case CF_STATE_RESTORE_MSG: |
||||
|
processor.process((CalculatedFieldStateRestoreMsg) msg); |
||||
|
break; |
||||
|
case CF_ENTITY_INIT_CF_MSG: |
||||
|
processor.process((EntityInitCalculatedFieldMsg) msg); |
||||
|
break; |
||||
|
case CF_ENTITY_DELETE_MSG: |
||||
|
processor.process((CalculatedFieldEntityDeleteMsg) msg); |
||||
|
break; |
||||
|
case CF_ENTITY_TELEMETRY_MSG: |
||||
|
processor.process((EntityCalculatedFieldTelemetryMsg) msg); |
||||
|
break; |
||||
|
case CF_LINKED_TELEMETRY_MSG: |
||||
|
processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); |
||||
|
break; |
||||
|
default: |
||||
|
return false; |
||||
|
} |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
void logProcessingException(Exception e) { |
||||
|
log.warn("[{}][{}] Processing failure", tenantId, processor.entityId, e); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,50 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.TbActor; |
||||
|
import org.thingsboard.server.actors.TbActorId; |
||||
|
import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; |
||||
|
import org.thingsboard.server.actors.TbEntityActorId; |
||||
|
import org.thingsboard.server.actors.device.DeviceActor; |
||||
|
import org.thingsboard.server.actors.service.ContextBasedCreator; |
||||
|
import org.thingsboard.server.common.data.id.DeviceId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
private final EntityId entityId; |
||||
|
|
||||
|
public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { |
||||
|
super(context); |
||||
|
this.tenantId = tenantId; |
||||
|
this.entityId = entityId; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public TbActorId createActorId() { |
||||
|
return new TbCalculatedFieldEntityActorId(entityId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public TbActor createActor() { |
||||
|
return new CalculatedFieldEntityActor(context, tenantId, entityId); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,44 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.MsgType; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
|
||||
|
@Data |
||||
|
public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
private final EntityId entityId; |
||||
|
private final TbCallback callback; |
||||
|
|
||||
|
public CalculatedFieldEntityDeleteMsg(TenantId tenantId, |
||||
|
EntityId entityId, |
||||
|
TbCallback callback) { |
||||
|
this.tenantId = tenantId; |
||||
|
this.entityId = entityId; |
||||
|
this.callback = callback; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public MsgType getMsgType() { |
||||
|
return MsgType.CF_ENTITY_DELETE_MSG; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,444 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import lombok.SneakyThrows; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.common.util.DebugModeUtil; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.TbActorCtx; |
||||
|
import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; |
||||
|
import org.thingsboard.server.common.data.AttributeScope; |
||||
|
import org.thingsboard.server.common.data.StringUtils; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.Argument; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.kv.StringDataEntry; |
||||
|
import org.thingsboard.server.common.data.msg.TbMsgType; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Collection; |
||||
|
import java.util.Collections; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.HashSet; |
||||
|
import java.util.LinkedList; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.Set; |
||||
|
import java.util.UUID; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* @author Andrew Shvayka |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { |
||||
|
// (1 for result persistence + 1 for the state persistence )
|
||||
|
public static final int CALLBACKS_PER_CF = 2; |
||||
|
|
||||
|
final TenantId tenantId; |
||||
|
final EntityId entityId; |
||||
|
final CalculatedFieldProcessingService cfService; |
||||
|
final CalculatedFieldStateService cfStateService; |
||||
|
final int partition; |
||||
|
|
||||
|
TbActorCtx ctx; |
||||
|
Map<CalculatedFieldId, CalculatedFieldState> states = new HashMap<>(); |
||||
|
|
||||
|
CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { |
||||
|
super(systemContext); |
||||
|
this.tenantId = tenantId; |
||||
|
this.entityId = entityId; |
||||
|
this.cfService = systemContext.getCalculatedFieldProcessingService(); |
||||
|
this.cfStateService = systemContext.getCalculatedFieldStateService(); |
||||
|
this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); |
||||
|
} |
||||
|
|
||||
|
void init(TbActorCtx ctx) { |
||||
|
this.ctx = ctx; |
||||
|
} |
||||
|
|
||||
|
public void process(CalculatedFieldPartitionChangeMsg msg) { |
||||
|
if (!msg.getPartitions()[partition]) { |
||||
|
log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); |
||||
|
ctx.stop(ctx.getSelf()); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void process(CalculatedFieldStateRestoreMsg msg) { |
||||
|
CalculatedFieldId cfId = msg.getId().cfId(); |
||||
|
log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); |
||||
|
if (msg.getState() != null) { |
||||
|
states.put(cfId, msg.getState()); |
||||
|
} else { |
||||
|
states.remove(cfId); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { |
||||
|
log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); |
||||
|
var ctx = msg.getCtx(); |
||||
|
if (msg.isForceReinit()) { |
||||
|
log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); |
||||
|
states.remove(ctx.getCfId()); |
||||
|
} |
||||
|
try { |
||||
|
var state = getOrInitState(ctx); |
||||
|
if (state.isSizeOk()) { |
||||
|
processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); |
||||
|
} else { |
||||
|
throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
if (e instanceof CalculatedFieldException cfe) { |
||||
|
throw cfe; |
||||
|
} |
||||
|
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void process(CalculatedFieldEntityDeleteMsg msg) { |
||||
|
log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); |
||||
|
if (this.entityId.equals(msg.getEntityId())) { |
||||
|
if (states.isEmpty()) { |
||||
|
msg.getCallback().onSuccess(); |
||||
|
} else { |
||||
|
MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); |
||||
|
states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); |
||||
|
ctx.stop(ctx.getSelf()); |
||||
|
} |
||||
|
} else { |
||||
|
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
||||
|
var state = states.remove(cfId); |
||||
|
if (state != null) { |
||||
|
cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); |
||||
|
} else { |
||||
|
msg.getCallback().onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { |
||||
|
log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); |
||||
|
var proto = msg.getProto(); |
||||
|
var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); |
||||
|
MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); |
||||
|
List<CalculatedFieldId> cfIdList = getCalculatedFieldIds(proto); |
||||
|
Set<CalculatedFieldId> cfIdSet = new HashSet<>(cfIdList); |
||||
|
for (var ctx : msg.getEntityIdFields()) { |
||||
|
process(ctx, proto, cfIdSet, cfIdList, callback); |
||||
|
} |
||||
|
for (var ctx : msg.getProfileIdFields()) { |
||||
|
process(ctx, proto, cfIdSet, cfIdList, callback); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) throws CalculatedFieldException { |
||||
|
log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); |
||||
|
var proto = msg.getProto(); |
||||
|
var ctx = msg.getCtx(); |
||||
|
var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); |
||||
|
try { |
||||
|
List<CalculatedFieldId> cfIds = getCalculatedFieldIds(proto); |
||||
|
if (cfIds.contains(ctx.getCfId())) { |
||||
|
callback.onSuccess(CALLBACKS_PER_CF); |
||||
|
} else { |
||||
|
if (proto.getTsDataCount() > 0) { |
||||
|
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} else if (proto.getAttrDataCount() > 0) { |
||||
|
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} else if (proto.getRemovedTsKeysCount() > 0) { |
||||
|
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} else if (proto.getRemovedAttrKeysCount() > 0) { |
||||
|
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} else { |
||||
|
callback.onSuccess(CALLBACKS_PER_CF); |
||||
|
} |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection<CalculatedFieldId> cfIds, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
||||
|
try { |
||||
|
if (cfIds.contains(ctx.getCfId())) { |
||||
|
callback.onSuccess(CALLBACKS_PER_CF); |
||||
|
} else { |
||||
|
if (proto.getTsDataCount() > 0) { |
||||
|
processTelemetry(ctx, proto, cfIdList, callback); |
||||
|
} else if (proto.getAttrDataCount() > 0) { |
||||
|
processAttributes(ctx, proto, cfIdList, callback); |
||||
|
} else if (proto.getRemovedTsKeysCount() > 0) { |
||||
|
processRemovedTelemetry(ctx, proto, cfIdList, callback); |
||||
|
} else if (proto.getRemovedAttrKeysCount() > 0) { |
||||
|
processRemovedAttributes(ctx, proto, cfIdList, callback); |
||||
|
} else { |
||||
|
callback.onSuccess(CALLBACKS_PER_CF); |
||||
|
} |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
if (e instanceof CalculatedFieldException cfe) { |
||||
|
throw cfe; |
||||
|
} |
||||
|
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
||||
|
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} |
||||
|
|
||||
|
private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
||||
|
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} |
||||
|
|
||||
|
private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
||||
|
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} |
||||
|
|
||||
|
private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { |
||||
|
processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithDefaultValue(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); |
||||
|
} |
||||
|
|
||||
|
private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback, |
||||
|
Map<String, ArgumentEntry> newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { |
||||
|
if (newArgValues.isEmpty()) { |
||||
|
log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); |
||||
|
callback.onSuccess(CALLBACKS_PER_CF); |
||||
|
} |
||||
|
CalculatedFieldState state = states.get(ctx.getCfId()); |
||||
|
boolean justRestored = false; |
||||
|
if (state == null) { |
||||
|
state = getOrInitState(ctx); |
||||
|
justRestored = true; |
||||
|
} |
||||
|
if (state.isSizeOk()) { |
||||
|
if (state.updateState(ctx, newArgValues) || justRestored) { |
||||
|
cfIdList = new ArrayList<>(cfIdList); |
||||
|
cfIdList.add(ctx.getCfId()); |
||||
|
processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); |
||||
|
} else { |
||||
|
callback.onSuccess(CALLBACKS_PER_CF); |
||||
|
} |
||||
|
} else { |
||||
|
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@SneakyThrows |
||||
|
private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { |
||||
|
CalculatedFieldState state = states.get(ctx.getCfId()); |
||||
|
if (state != null) { |
||||
|
return state; |
||||
|
} else { |
||||
|
ListenableFuture<CalculatedFieldState> stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); |
||||
|
// Ugly but necessary. We do not expect to often fetch data from DB. Only once per <Entity, CalculatedField> pair lifetime.
|
||||
|
// This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low.
|
||||
|
// Alternatively, we can fetch the state outside the actor system and push separate command to create this actor,
|
||||
|
// but this will significantly complicate the code.
|
||||
|
state = stateFuture.get(1, TimeUnit.MINUTES); |
||||
|
state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); |
||||
|
states.put(ctx.getCfId(), state); |
||||
|
} |
||||
|
return state; |
||||
|
} |
||||
|
|
||||
|
private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { |
||||
|
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); |
||||
|
boolean stateSizeChecked = false; |
||||
|
try { |
||||
|
if (ctx.isInitialized() && state.isReady()) { |
||||
|
CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); |
||||
|
state.checkStateSize(ctxId, ctx.getMaxStateSize()); |
||||
|
stateSizeChecked = true; |
||||
|
if (state.isSizeOk()) { |
||||
|
cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); |
||||
|
if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { |
||||
|
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); |
||||
|
} finally { |
||||
|
if (!stateSizeChecked) { |
||||
|
state.checkStateSize(ctxId, ctx.getMaxStateSize()); |
||||
|
} |
||||
|
if (state.isSizeOk()) { |
||||
|
cfStateService.persistState(ctxId, state, callback); |
||||
|
} else { |
||||
|
removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException { |
||||
|
// We remove the state, but remember that it is over-sized in a local map.
|
||||
|
cfStateService.removeState(ctxId, new TbCallback() { |
||||
|
@Override |
||||
|
public void onSuccess() { |
||||
|
callback.onFailure(ex); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
callback.onFailure(ex); |
||||
|
} |
||||
|
}); |
||||
|
throw ex; |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) { |
||||
|
return mapToArguments(ctx.getMainEntityArguments(), data); |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List<TsKvProto> data) { |
||||
|
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
||||
|
if (argNames.isEmpty()) { |
||||
|
return Collections.emptyMap(); |
||||
|
} |
||||
|
return mapToArguments(argNames, data); |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, List<TsKvProto> data) { |
||||
|
if (argNames.isEmpty()) { |
||||
|
return Collections.emptyMap(); |
||||
|
} |
||||
|
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
||||
|
for (TsKvProto item : data) { |
||||
|
ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); |
||||
|
String argName = argNames.get(key); |
||||
|
if (argName != null) { |
||||
|
arguments.put(argName, new SingleValueArgumentEntry(item)); |
||||
|
} |
||||
|
key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); |
||||
|
argName = argNames.get(key); |
||||
|
if (argName != null) { |
||||
|
arguments.put(argName, new SingleValueArgumentEntry(item)); |
||||
|
} |
||||
|
} |
||||
|
return arguments; |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
||||
|
return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
||||
|
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
||||
|
if (argNames.isEmpty()) { |
||||
|
return Collections.emptyMap(); |
||||
|
} |
||||
|
return mapToArguments(argNames, scope, attrDataList); |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArguments(Map<ReferencedEntityKey, String> argNames, AttributeScopeProto scope, List<AttributeValueProto> attrDataList) { |
||||
|
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
||||
|
for (AttributeValueProto item : attrDataList) { |
||||
|
ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); |
||||
|
String argName = argNames.get(key); |
||||
|
if (argName != null) { |
||||
|
arguments.put(argName, new SingleValueArgumentEntry(item)); |
||||
|
} |
||||
|
} |
||||
|
return arguments; |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List<String> removedAttrKeys) { |
||||
|
var argNames = ctx.getLinkedEntityArguments().get(entityId); |
||||
|
if (argNames.isEmpty()) { |
||||
|
return Collections.emptyMap(); |
||||
|
} |
||||
|
return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List<String> removedAttrKeys) { |
||||
|
return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArgumentsWithDefaultValue(Map<ReferencedEntityKey, String> argNames, Map<String, Argument> configArguments, AttributeScopeProto scope, List<String> removedAttrKeys) { |
||||
|
Map<String, ArgumentEntry> arguments = new HashMap<>(); |
||||
|
for (String removedKey : removedAttrKeys) { |
||||
|
ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); |
||||
|
String argName = argNames.get(key); |
||||
|
if (argName != null) { |
||||
|
Argument argument = configArguments.get(argName); |
||||
|
String defaultValue = (argument != null) ? argument.getDefaultValue() : null; |
||||
|
arguments.put(argName, StringUtils.isNotEmpty(defaultValue) |
||||
|
? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) |
||||
|
: new SingleValueArgumentEntry()); |
||||
|
|
||||
|
} |
||||
|
} |
||||
|
return arguments; |
||||
|
} |
||||
|
|
||||
|
private Map<String, ArgumentEntry> mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List<String> removedTelemetryKeys) { |
||||
|
Map<String, Argument> deletedArguments = ctx.getArguments().entrySet().stream() |
||||
|
.filter(entry -> removedTelemetryKeys.contains(entry.getKey())) |
||||
|
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); |
||||
|
|
||||
|
Map<String, ArgumentEntry> fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); |
||||
|
|
||||
|
fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); |
||||
|
return fetchedArgs; |
||||
|
} |
||||
|
|
||||
|
private static List<CalculatedFieldId> getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { |
||||
|
List<CalculatedFieldId> cfIds = new LinkedList<>(); |
||||
|
for (var cfId : proto.getPreviousCalculatedFieldsList()) { |
||||
|
cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); |
||||
|
} |
||||
|
return cfIds; |
||||
|
} |
||||
|
|
||||
|
private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { |
||||
|
if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { |
||||
|
return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { |
||||
|
if (!proto.getTbMsgType().isEmpty()) { |
||||
|
return TbMsgType.valueOf(proto.getTbMsgType()); |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,40 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Builder; |
||||
|
import lombok.Getter; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.msg.TbMsgType; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
|
||||
|
import java.util.Map; |
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@Getter |
||||
|
@Builder |
||||
|
public class CalculatedFieldException extends Exception { |
||||
|
|
||||
|
private final CalculatedFieldCtx ctx; |
||||
|
private final EntityId eventEntity; |
||||
|
private final UUID msgId; |
||||
|
private final TbMsgType msgType; |
||||
|
private Map<String, ArgumentEntry> arguments; |
||||
|
private String errorMessage; |
||||
|
private Exception cause; |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,40 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.MsgType; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
|
||||
|
@Data |
||||
|
public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
private final EntityId entityId; |
||||
|
private final CalculatedFieldLinkedTelemetryMsgProto proto; |
||||
|
private final TbCallback callback; |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public MsgType getMsgType() { |
||||
|
return MsgType.CF_LINKED_TELEMETRY_MSG; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,90 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.TbActorCtx; |
||||
|
import org.thingsboard.server.actors.TbActorException; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
||||
|
|
||||
|
/** |
||||
|
* Created by ashvayka on 15.03.18. |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { |
||||
|
|
||||
|
private final CalculatedFieldManagerMessageProcessor processor; |
||||
|
|
||||
|
public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { |
||||
|
super(systemContext, tenantId); |
||||
|
this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void init(TbActorCtx ctx) throws TbActorException { |
||||
|
super.init(ctx); |
||||
|
log.debug("[{}] Starting CF manager actor.", processor.tenantId); |
||||
|
try { |
||||
|
processor.init(ctx); |
||||
|
log.debug("[{}] CF manager actor started.", processor.tenantId); |
||||
|
} catch (Exception e) { |
||||
|
log.warn("[{}] Unknown failure", processor.tenantId, e); |
||||
|
throw new TbActorException("Failed to initialize manager actor", e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { |
||||
|
switch (msg.getMsgType()) { |
||||
|
case CF_PARTITIONS_CHANGE_MSG: |
||||
|
processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); |
||||
|
break; |
||||
|
case CF_INIT_MSG: |
||||
|
processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); |
||||
|
break; |
||||
|
case CF_LINK_INIT_MSG: |
||||
|
processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); |
||||
|
break; |
||||
|
case CF_STATE_RESTORE_MSG: |
||||
|
processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); |
||||
|
break; |
||||
|
case CF_ENTITY_LIFECYCLE_MSG: |
||||
|
processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); |
||||
|
break; |
||||
|
case CF_TELEMETRY_MSG: |
||||
|
processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); |
||||
|
break; |
||||
|
case CF_LINKED_TELEMETRY_MSG: |
||||
|
processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); |
||||
|
break; |
||||
|
default: |
||||
|
return false; |
||||
|
} |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
void logProcessingException(Exception e) { |
||||
|
log.warn("[{}] Processing failure", tenantId, e); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,46 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.TbActor; |
||||
|
import org.thingsboard.server.actors.TbActorId; |
||||
|
import org.thingsboard.server.actors.TbEntityActorId; |
||||
|
import org.thingsboard.server.actors.TbStringActorId; |
||||
|
import org.thingsboard.server.actors.service.ContextBasedCreator; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
|
||||
|
public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { |
||||
|
super(context); |
||||
|
this.tenantId = tenantId; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public TbActorId createActorId() { |
||||
|
return new TbStringActorId("CFM|" + tenantId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public TbActor createActor() { |
||||
|
return new CalculatedFieldManagerActor(context, tenantId); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,468 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.TbActorCtx; |
||||
|
import org.thingsboard.server.actors.TbActorRef; |
||||
|
import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; |
||||
|
import org.thingsboard.server.actors.service.DefaultActorService; |
||||
|
import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
||||
|
import org.thingsboard.server.common.data.id.AssetId; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.DeviceId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
||||
|
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
||||
|
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
||||
|
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Collections; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.concurrent.CopyOnWriteArrayList; |
||||
|
|
||||
|
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* @author Andrew Shvayka |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { |
||||
|
|
||||
|
private final Map<CalculatedFieldId, CalculatedFieldCtx> calculatedFields = new HashMap<>(); |
||||
|
private final Map<EntityId, List<CalculatedFieldCtx>> entityIdCalculatedFields = new HashMap<>(); |
||||
|
private final Map<EntityId, List<CalculatedFieldLink>> entityIdCalculatedFieldLinks = new HashMap<>(); |
||||
|
|
||||
|
private final CalculatedFieldProcessingService cfExecService; |
||||
|
private final CalculatedFieldStateService cfStateService; |
||||
|
private final CalculatedFieldEntityProfileCache cfEntityCache; |
||||
|
private final CalculatedFieldService cfDaoService; |
||||
|
private final TbAssetProfileCache assetProfileCache; |
||||
|
private final TbDeviceProfileCache deviceProfileCache; |
||||
|
protected final TenantId tenantId; |
||||
|
|
||||
|
protected TbActorCtx ctx; |
||||
|
|
||||
|
CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { |
||||
|
super(systemContext); |
||||
|
this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); |
||||
|
this.cfExecService = systemContext.getCalculatedFieldProcessingService(); |
||||
|
this.cfStateService = systemContext.getCalculatedFieldStateService(); |
||||
|
this.cfDaoService = systemContext.getCalculatedFieldService(); |
||||
|
this.assetProfileCache = systemContext.getAssetProfileCache(); |
||||
|
this.deviceProfileCache = systemContext.getDeviceProfileCache(); |
||||
|
this.tenantId = tenantId; |
||||
|
} |
||||
|
|
||||
|
void init(TbActorCtx ctx) { |
||||
|
this.ctx = ctx; |
||||
|
} |
||||
|
|
||||
|
public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException { |
||||
|
log.info("[{}] Processing CF init message.", msg.getCf().getId()); |
||||
|
var cf = msg.getCf(); |
||||
|
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
||||
|
try { |
||||
|
cfCtx.init(); |
||||
|
} catch (Exception e) { |
||||
|
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
||||
|
} |
||||
|
calculatedFields.put(cf.getId(), cfCtx); |
||||
|
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
||||
|
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
||||
|
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); |
||||
|
msg.getCallback().onSuccess(); |
||||
|
} |
||||
|
|
||||
|
public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { |
||||
|
log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); |
||||
|
var link = msg.getLink(); |
||||
|
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
||||
|
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
||||
|
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); |
||||
|
msg.getCallback().onSuccess(); |
||||
|
} |
||||
|
|
||||
|
public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { |
||||
|
var cfId = msg.getId().cfId(); |
||||
|
var calculatedField = calculatedFields.get(cfId); |
||||
|
|
||||
|
if (calculatedField != null) { |
||||
|
msg.getState().setRequiredArguments(calculatedField.getArgNames()); |
||||
|
log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); |
||||
|
getOrCreateActor(msg.getId().entityId()).tell(msg); |
||||
|
} else { |
||||
|
cfStateService.removeState(msg.getId(), msg.getCallback()); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { |
||||
|
log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); |
||||
|
var entityType = msg.getData().getEntityId().getEntityType(); |
||||
|
var event = msg.getData().getEvent(); |
||||
|
switch (entityType) { |
||||
|
case CALCULATED_FIELD: { |
||||
|
switch (event) { |
||||
|
case CREATED: |
||||
|
onCfCreated(msg.getData(), msg.getCallback()); |
||||
|
break; |
||||
|
case UPDATED: |
||||
|
onCfUpdated(msg.getData(), msg.getCallback()); |
||||
|
break; |
||||
|
case DELETED: |
||||
|
onCfDeleted(msg.getData(), msg.getCallback()); |
||||
|
break; |
||||
|
default: |
||||
|
msg.getCallback().onSuccess(); |
||||
|
break; |
||||
|
} |
||||
|
break; |
||||
|
} |
||||
|
case DEVICE: |
||||
|
case ASSET: { |
||||
|
switch (event) { |
||||
|
case CREATED: |
||||
|
onEntityCreated(msg.getData(), msg.getCallback()); |
||||
|
break; |
||||
|
case UPDATED: |
||||
|
onEntityUpdated(msg.getData(), msg.getCallback()); |
||||
|
break; |
||||
|
case DELETED: |
||||
|
onEntityDeleted(msg.getData(), msg.getCallback()); |
||||
|
break; |
||||
|
default: |
||||
|
msg.getCallback().onSuccess(); |
||||
|
break; |
||||
|
} |
||||
|
break; |
||||
|
} |
||||
|
default: { |
||||
|
msg.getCallback().onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { |
||||
|
EntityId entityId = msg.getEntityId(); |
||||
|
EntityId profileId = getProfileId(tenantId, entityId); |
||||
|
cfEntityCache.add(tenantId, profileId, entityId); |
||||
|
var entityIdFields = getCalculatedFieldsByEntityId(entityId); |
||||
|
var profileIdFields = getCalculatedFieldsByEntityId(profileId); |
||||
|
var fieldsCount = entityIdFields.size() + profileIdFields.size(); |
||||
|
if (fieldsCount > 0) { |
||||
|
MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); |
||||
|
entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
||||
|
profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
||||
|
} else { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { |
||||
|
if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { |
||||
|
cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); |
||||
|
var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); |
||||
|
var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); |
||||
|
var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); |
||||
|
if (fieldsCount > 0) { |
||||
|
MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); |
||||
|
var entityId = msg.getEntityId(); |
||||
|
oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); |
||||
|
newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); |
||||
|
} else { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { |
||||
|
cfEntityCache.evict(tenantId, msg.getEntityId()); |
||||
|
log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); |
||||
|
getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); |
||||
|
} |
||||
|
|
||||
|
private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { |
||||
|
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
||||
|
if (calculatedFields.containsKey(cfId)) { |
||||
|
log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); |
||||
|
callback.onSuccess(); |
||||
|
} else { |
||||
|
var cf = cfDaoService.findById(msg.getTenantId(), cfId); |
||||
|
if (cf == null) { |
||||
|
log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); |
||||
|
callback.onSuccess(); |
||||
|
} else { |
||||
|
var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
||||
|
try { |
||||
|
cfCtx.init(); |
||||
|
} catch (Exception e) { |
||||
|
throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
||||
|
} |
||||
|
calculatedFields.put(cf.getId(), cfCtx); |
||||
|
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
||||
|
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
||||
|
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); |
||||
|
addLinks(cf); |
||||
|
initCf(cfCtx, callback, false); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { |
||||
|
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
||||
|
var oldCfCtx = calculatedFields.get(cfId); |
||||
|
if (oldCfCtx == null) { |
||||
|
onCfCreated(msg, callback); |
||||
|
} else { |
||||
|
var newCf = cfDaoService.findById(msg.getTenantId(), cfId); |
||||
|
if (newCf == null) { |
||||
|
log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); |
||||
|
callback.onSuccess(); |
||||
|
} else { |
||||
|
var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); |
||||
|
try { |
||||
|
newCfCtx.init(); |
||||
|
} catch (Exception e) { |
||||
|
throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); |
||||
|
} |
||||
|
calculatedFields.put(newCf.getId(), newCfCtx); |
||||
|
List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); |
||||
|
List<CalculatedFieldCtx> newCfList = new CopyOnWriteArrayList<>(); |
||||
|
boolean found = false; |
||||
|
for (CalculatedFieldCtx oldCtx : oldCfList) { |
||||
|
if (oldCtx.getCfId().equals(newCf.getId())) { |
||||
|
newCfList.add(newCfCtx); |
||||
|
found = true; |
||||
|
} else { |
||||
|
newCfList.add(oldCtx); |
||||
|
} |
||||
|
} |
||||
|
if (!found) { |
||||
|
newCfList.add(newCfCtx); |
||||
|
} |
||||
|
entityIdCalculatedFields.put(newCf.getEntityId(), newCfList); |
||||
|
|
||||
|
deleteLinks(oldCfCtx); |
||||
|
addLinks(newCf); |
||||
|
|
||||
|
// We use copy on write lists to safely pass the reference to another actor for the iteration.
|
||||
|
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
|
||||
|
var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); |
||||
|
if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { |
||||
|
initCf(newCfCtx, callback, stateChanges); |
||||
|
} else { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { |
||||
|
var cfId = new CalculatedFieldId(msg.getEntityId().getId()); |
||||
|
var cfCtx = calculatedFields.remove(cfId); |
||||
|
if (cfCtx == null) { |
||||
|
log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); |
||||
|
callback.onSuccess(); |
||||
|
} else { |
||||
|
entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); |
||||
|
deleteLinks(cfCtx); |
||||
|
|
||||
|
EntityId entityId = cfCtx.getEntityId(); |
||||
|
EntityType entityType = cfCtx.getEntityId().getEntityType(); |
||||
|
if (isProfileEntity(entityType)) { |
||||
|
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); |
||||
|
if (!entityIds.isEmpty()) { |
||||
|
//TODO: no need to do this if we cache all created actors and know which one belong to us;
|
||||
|
var multiCallback = new MultipleTbCallback(entityIds.size(), callback); |
||||
|
entityIds.forEach(id -> deleteCfForEntity(id, cfId, multiCallback)); |
||||
|
} else { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} else { |
||||
|
deleteCfForEntity(entityId, cfId, callback); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { |
||||
|
EntityId entityId = msg.getEntityId(); |
||||
|
log.info("Received telemetry msg from entity [{}]", entityId); |
||||
|
// 2 = 1 for CF processing + 1 for links processing
|
||||
|
MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); |
||||
|
// process all cfs related to entity, or it's profile;
|
||||
|
var entityIdFields = getCalculatedFieldsByEntityId(entityId); |
||||
|
var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); |
||||
|
if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { |
||||
|
log.info("Pushing telemetry msg to specific actor [{}]", entityId); |
||||
|
getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); |
||||
|
} else { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
// process all links (if any);
|
||||
|
List<CalculatedFieldEntityCtxId> linkedCalculatedFields = filterCalculatedFieldLinks(msg); |
||||
|
var linksSize = linkedCalculatedFields.size(); |
||||
|
if (linksSize > 0) { |
||||
|
cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); |
||||
|
} else { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { |
||||
|
EntityId sourceEntityId = msg.getEntityId(); |
||||
|
log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); |
||||
|
var proto = msg.getProto(); |
||||
|
var linksList = proto.getLinksList(); |
||||
|
for (var linkProto : linksList) { |
||||
|
var link = fromProto(linkProto); |
||||
|
var targetEntityId = link.entityId(); |
||||
|
var targetEntityType = targetEntityId.getEntityType(); |
||||
|
var cf = calculatedFields.get(link.cfId()); |
||||
|
if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { |
||||
|
// iterate over all entities that belong to profile and push the message for corresponding CF
|
||||
|
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); |
||||
|
if (!entityIds.isEmpty()) { |
||||
|
MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); |
||||
|
var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); |
||||
|
entityIds.forEach(entityId -> { |
||||
|
log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); |
||||
|
getOrCreateActor(entityId).tell(newMsg); |
||||
|
}); |
||||
|
} else { |
||||
|
msg.getCallback().onSuccess(); |
||||
|
} |
||||
|
} else { |
||||
|
log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); |
||||
|
var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); |
||||
|
getOrCreateActor(targetEntityId).tell(newMsg); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private List<CalculatedFieldEntityCtxId> filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { |
||||
|
EntityId entityId = msg.getEntityId(); |
||||
|
var proto = msg.getProto(); |
||||
|
List<CalculatedFieldEntityCtxId> result = new ArrayList<>(); |
||||
|
for (var link : getCalculatedFieldLinksByEntityId(entityId)) { |
||||
|
CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); |
||||
|
if (ctx.linkMatches(entityId, proto)) { |
||||
|
result.add(ctx.toCalculatedFieldEntityCtxId()); |
||||
|
} |
||||
|
} |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
private List<CalculatedFieldCtx> getCalculatedFieldsByEntityId(EntityId entityId) { |
||||
|
if (entityId == null) { |
||||
|
return Collections.emptyList(); |
||||
|
} |
||||
|
var result = entityIdCalculatedFields.get(entityId); |
||||
|
if (result == null) { |
||||
|
result = Collections.emptyList(); |
||||
|
} |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
private List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId) { |
||||
|
if (entityId == null) { |
||||
|
return Collections.emptyList(); |
||||
|
} |
||||
|
var result = entityIdCalculatedFieldLinks.get(entityId); |
||||
|
if (result == null) { |
||||
|
result = Collections.emptyList(); |
||||
|
} |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { |
||||
|
EntityId entityId = cfCtx.getEntityId(); |
||||
|
EntityType entityType = cfCtx.getEntityId().getEntityType(); |
||||
|
if (isProfileEntity(entityType)) { |
||||
|
var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); |
||||
|
if (!entityIds.isEmpty()) { |
||||
|
var multiCallback = new MultipleTbCallback(entityIds.size(), callback); |
||||
|
entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); |
||||
|
} else { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} else { |
||||
|
initCfForEntity(entityId, cfCtx, forceStateReinit, callback); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { |
||||
|
log.info("Pushing delete CF msg to specific actor [{}]", entityId); |
||||
|
getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); |
||||
|
} |
||||
|
|
||||
|
private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { |
||||
|
log.info("Pushing entity init CF msg to specific actor [{}]", entityId); |
||||
|
getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); |
||||
|
} |
||||
|
|
||||
|
private static boolean isProfileEntity(EntityType entityType) { |
||||
|
return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); |
||||
|
} |
||||
|
|
||||
|
private EntityId getProfileId(TenantId tenantId, EntityId entityId) { |
||||
|
return switch (entityId.getEntityType()) { |
||||
|
case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); |
||||
|
case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); |
||||
|
default -> null; |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
private TbActorRef getOrCreateActor(EntityId entityId) { |
||||
|
return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), |
||||
|
() -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, |
||||
|
() -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), |
||||
|
() -> true); |
||||
|
} |
||||
|
|
||||
|
private void addLinks(CalculatedField newCf) { |
||||
|
var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); |
||||
|
newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link)); |
||||
|
} |
||||
|
|
||||
|
private void deleteLinks(CalculatedFieldCtx cfCtx) { |
||||
|
var oldCf = cfCtx.getCalculatedField(); |
||||
|
var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); |
||||
|
oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).remove(link)); |
||||
|
} |
||||
|
|
||||
|
public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { |
||||
|
ctx.broadcastToChildren(msg, true); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,40 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.MsgType; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
||||
|
|
||||
|
@Data |
||||
|
public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { |
||||
|
|
||||
|
private final CalculatedFieldEntityCtxId id; |
||||
|
private final CalculatedFieldState state; |
||||
|
|
||||
|
@Override |
||||
|
public MsgType getMsgType() { |
||||
|
return MsgType.CF_STATE_RESTORE_MSG; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public TenantId getTenantId() { |
||||
|
return id.tenantId(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,39 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.MsgType; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
|
||||
|
@Data |
||||
|
public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
private final EntityId entityId; |
||||
|
private final CalculatedFieldTelemetryMsgProto proto; |
||||
|
private final TbCallback callback; |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public MsgType getMsgType() { |
||||
|
return MsgType.CF_TELEMETRY_MSG; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,42 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.MsgType; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
@Data |
||||
|
public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
private final EntityId entityId; |
||||
|
private final CalculatedFieldTelemetryMsgProto proto; |
||||
|
private final CalculatedFieldCtx ctx; |
||||
|
private final TbCallback callback; |
||||
|
|
||||
|
@Override |
||||
|
public MsgType getMsgType() { |
||||
|
return MsgType.CF_LINKED_TELEMETRY_MSG; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,56 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.MsgType; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
@Data |
||||
|
public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
private final EntityId entityId; |
||||
|
private final CalculatedFieldTelemetryMsgProto proto; |
||||
|
// Both lists are effectively immutable in CalculatedFieldManagerMessageProcessor and must stay so.
|
||||
|
private final List<CalculatedFieldCtx> entityIdFields; |
||||
|
private final List<CalculatedFieldCtx> profileIdFields; |
||||
|
private final TbCallback callback; |
||||
|
|
||||
|
public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, |
||||
|
List<CalculatedFieldCtx> entityIdFields, |
||||
|
List<CalculatedFieldCtx> profileIdFields, |
||||
|
TbCallback callback) { |
||||
|
this.tenantId = msg.getTenantId(); |
||||
|
this.entityId = msg.getEntityId(); |
||||
|
this.proto = msg.getProto(); |
||||
|
this.entityIdFields = entityIdFields; |
||||
|
this.profileIdFields = profileIdFields; |
||||
|
this.callback = callback; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public MsgType getMsgType() { |
||||
|
return MsgType.CF_ENTITY_TELEMETRY_MSG; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,41 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.MsgType; |
||||
|
import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
@Data |
||||
|
public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { |
||||
|
|
||||
|
private final TenantId tenantId; |
||||
|
private final CalculatedFieldCtx ctx; |
||||
|
private final TbCallback callback; |
||||
|
private final boolean forceReinit; |
||||
|
|
||||
|
@Override |
||||
|
public MsgType getMsgType() { |
||||
|
return MsgType.CF_ENTITY_INIT_CF_MSG; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,56 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.actors.calculatedField; |
||||
|
|
||||
|
import lombok.Getter; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
import java.util.concurrent.atomic.AtomicInteger; |
||||
|
|
||||
|
@Slf4j |
||||
|
public class MultipleTbCallback implements TbCallback { |
||||
|
@Getter |
||||
|
private final UUID id; |
||||
|
private final AtomicInteger counter; |
||||
|
private final TbCallback callback; |
||||
|
|
||||
|
public MultipleTbCallback(int count, TbCallback callback) { |
||||
|
id = UUID.randomUUID(); |
||||
|
this.counter = new AtomicInteger(count); |
||||
|
this.callback = callback; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onSuccess() { |
||||
|
onSuccess(1); |
||||
|
} |
||||
|
|
||||
|
public void onSuccess(int number) { |
||||
|
log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); |
||||
|
if (counter.addAndGet(-number) <= 0) { |
||||
|
log.trace("[{}][{}] Done.", id, callback.getId()); |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
log.warn("[{}][{}] onFailure.", id, callback.getId()); |
||||
|
callback.onFailure(t); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,283 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.controller; |
||||
|
|
||||
|
import com.fasterxml.jackson.core.type.TypeReference; |
||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
import com.fasterxml.jackson.databind.node.ObjectNode; |
||||
|
import io.swagger.v3.oas.annotations.Parameter; |
||||
|
import io.swagger.v3.oas.annotations.media.Schema; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.http.HttpStatus; |
||||
|
import org.springframework.security.access.prepost.PreAuthorize; |
||||
|
import org.springframework.web.bind.annotation.PathVariable; |
||||
|
import org.springframework.web.bind.annotation.RequestBody; |
||||
|
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
import org.springframework.web.bind.annotation.RequestMethod; |
||||
|
import org.springframework.web.bind.annotation.RequestParam; |
||||
|
import org.springframework.web.bind.annotation.ResponseBody; |
||||
|
import org.springframework.web.bind.annotation.ResponseStatus; |
||||
|
import org.springframework.web.bind.annotation.RestController; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfArg; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfCtx; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; |
||||
|
import org.thingsboard.script.api.tbel.TbelInvokeService; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.EventInfo; |
||||
|
import org.thingsboard.server.common.data.HasTenantId; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
||||
|
import org.thingsboard.server.common.data.event.EventType; |
||||
|
import org.thingsboard.server.common.data.exception.ThingsboardException; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.EntityIdFactory; |
||||
|
import org.thingsboard.server.common.data.id.HasId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.page.PageData; |
||||
|
import org.thingsboard.server.common.data.page.PageLink; |
||||
|
import org.thingsboard.server.config.annotations.ApiOperation; |
||||
|
import org.thingsboard.server.dao.event.EventService; |
||||
|
import org.thingsboard.server.queue.util.TbCoreComponent; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; |
||||
|
import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; |
||||
|
import org.thingsboard.server.service.security.model.SecurityUser; |
||||
|
import org.thingsboard.server.service.security.permission.Operation; |
||||
|
import org.thingsboard.server.service.security.permission.Resource; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Collections; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.Objects; |
||||
|
import java.util.Optional; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; |
||||
|
import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; |
||||
|
|
||||
|
@RestController |
||||
|
@TbCoreComponent |
||||
|
@RequestMapping("/api") |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
public class CalculatedFieldController extends BaseController { |
||||
|
|
||||
|
private final TbCalculatedFieldService tbCalculatedFieldService; |
||||
|
private final EventService eventService; |
||||
|
private final TbelInvokeService tbelInvokeService; |
||||
|
|
||||
|
public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; |
||||
|
|
||||
|
public static final int TIMEOUT = 20; |
||||
|
|
||||
|
private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" |
||||
|
+ MARKDOWN_CODE_BLOCK_START |
||||
|
+ "{\n" + |
||||
|
" \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" + |
||||
|
" \"arguments\": {\n" + |
||||
|
" \"temperature\": {\n" + |
||||
|
" \"type\": \"TS_ROLLING\",\n" + |
||||
|
" \"timeWindow\": {\n" + |
||||
|
" \"startTs\": 1739775630002,\n" + |
||||
|
" \"endTs\": 65432211,\n" + |
||||
|
" \"limit\": 5\n" + |
||||
|
" },\n" + |
||||
|
" \"values\": [\n" + |
||||
|
" { \"ts\": 1739775639851, \"value\": 23 },\n" + |
||||
|
" { \"ts\": 1739775664561, \"value\": 43 },\n" + |
||||
|
" { \"ts\": 1739775713079, \"value\": 15 },\n" + |
||||
|
" { \"ts\": 1739775999522, \"value\": 34 },\n" + |
||||
|
" { \"ts\": 1739776228452, \"value\": 22 }\n" + |
||||
|
" ]\n" + |
||||
|
" },\n" + |
||||
|
" \"humidity\": { \"type\": \"SINGLE_VALUE\", \"ts\": 1739776478057, \"value\": 23 }\n" + |
||||
|
" }\n" + |
||||
|
"}" |
||||
|
+ MARKDOWN_CODE_BLOCK_END |
||||
|
+ "\n\n Expected result JSON contains \"output\" and \"error\"."; |
||||
|
|
||||
|
@ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", |
||||
|
notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + |
||||
|
"The newly created Calculated Field Id will be present in the response. " + |
||||
|
"Specify existing Calculated Field Id to update the calculated field. " + |
||||
|
"Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + |
||||
|
"Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " |
||||
|
+ TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) |
||||
|
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
||||
|
@RequestMapping(value = "/calculatedField", method = RequestMethod.POST) |
||||
|
@ResponseBody |
||||
|
public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") |
||||
|
@RequestBody CalculatedField calculatedField) throws Exception { |
||||
|
calculatedField.setTenantId(getTenantId()); |
||||
|
checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); |
||||
|
checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); |
||||
|
checkReferencedEntities(calculatedField.getConfiguration(), getCurrentUser()); |
||||
|
return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); |
||||
|
} |
||||
|
|
||||
|
@ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", |
||||
|
notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." |
||||
|
) |
||||
|
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
||||
|
@RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) |
||||
|
@ResponseBody |
||||
|
public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { |
||||
|
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
||||
|
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
||||
|
CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); |
||||
|
checkNotNull(calculatedField); |
||||
|
checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); |
||||
|
return calculatedField; |
||||
|
} |
||||
|
|
||||
|
@ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", |
||||
|
notes = "Fetch the Calculated Fields based on the provided Entity Id." |
||||
|
) |
||||
|
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
||||
|
@RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) |
||||
|
@ResponseBody |
||||
|
public PageData<CalculatedField> getCalculatedFieldsByEntityId( |
||||
|
@Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, |
||||
|
@Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, |
||||
|
@Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, |
||||
|
@Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, |
||||
|
@Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, |
||||
|
@Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, |
||||
|
@Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { |
||||
|
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); |
||||
|
checkParameter("entityId", entityIdStr); |
||||
|
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); |
||||
|
checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); |
||||
|
return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); |
||||
|
} |
||||
|
|
||||
|
@ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", |
||||
|
notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) |
||||
|
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
||||
|
@RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) |
||||
|
@ResponseStatus(value = HttpStatus.OK) |
||||
|
public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws Exception { |
||||
|
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
||||
|
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
||||
|
CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); |
||||
|
checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); |
||||
|
tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); |
||||
|
} |
||||
|
|
||||
|
@ApiOperation(value = "Get latest calculated field debug event (getLatestCalculatedFieldDebugEvent)", |
||||
|
notes = "Gets latest calculated field debug event for specified calculated field id. " + |
||||
|
"Referencing non-existing calculated field id will cause an error. " + TENANT_AUTHORITY_PARAGRAPH) |
||||
|
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") |
||||
|
@RequestMapping(value = "/calculatedField/{calculatedFieldId}/debug", method = RequestMethod.GET) |
||||
|
@ResponseBody |
||||
|
public JsonNode getLatestCalculatedFieldDebugEvent(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { |
||||
|
checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); |
||||
|
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); |
||||
|
CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.READ); |
||||
|
checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); |
||||
|
TenantId tenantId = getCurrentUser().getTenantId(); |
||||
|
return Optional.ofNullable(eventService.findLatestEvents(tenantId, calculatedFieldId, EventType.DEBUG_CALCULATED_FIELD, 1)) |
||||
|
.flatMap(events -> events.stream().map(EventInfo::getBody).findFirst()) |
||||
|
.orElse(null); |
||||
|
} |
||||
|
|
||||
|
@ApiOperation(value = "Test Script expression", |
||||
|
notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) |
||||
|
@PreAuthorize("hasAuthority('TENANT_ADMIN')") |
||||
|
@RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) |
||||
|
@ResponseBody |
||||
|
public JsonNode testScript( |
||||
|
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") |
||||
|
@RequestBody JsonNode inputParams) { |
||||
|
String expression = inputParams.get("expression").asText(); |
||||
|
Map<String, TbelCfArg> arguments = Objects.requireNonNullElse( |
||||
|
JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference<>() { |
||||
|
}), |
||||
|
Collections.emptyMap() |
||||
|
); |
||||
|
|
||||
|
ArrayList<String> ctxAndArgNames = new ArrayList<>(arguments.size() + 1); |
||||
|
ctxAndArgNames.add("ctx"); |
||||
|
ctxAndArgNames.addAll(arguments.keySet()); |
||||
|
|
||||
|
String output = ""; |
||||
|
String errorText = ""; |
||||
|
|
||||
|
try { |
||||
|
if (tbelInvokeService == null) { |
||||
|
throw new IllegalArgumentException("TBEL script engine is disabled!"); |
||||
|
} |
||||
|
|
||||
|
CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( |
||||
|
getTenantId(), |
||||
|
tbelInvokeService, |
||||
|
expression, |
||||
|
ctxAndArgNames.toArray(String[]::new) |
||||
|
); |
||||
|
|
||||
|
|
||||
|
Object[] args = new Object[ctxAndArgNames.size()]; |
||||
|
args[0] = new TbelCfCtx(arguments); |
||||
|
for (int i = 1; i < ctxAndArgNames.size(); i++) { |
||||
|
var arg = arguments.get(ctxAndArgNames.get(i)); |
||||
|
if (arg instanceof TbelCfSingleValueArg svArg) { |
||||
|
args[i] = svArg.getValue(); |
||||
|
} else { |
||||
|
args[i] = arg; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); |
||||
|
output = JacksonUtil.toString(json); |
||||
|
} catch (Exception e) { |
||||
|
log.error("Error evaluating expression", e); |
||||
|
errorText = e.getMessage(); |
||||
|
} |
||||
|
|
||||
|
ObjectNode result = JacksonUtil.newObjectNode(); |
||||
|
result.put("output", output); |
||||
|
result.put("error", errorText); |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
private <E extends HasId<I> & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { |
||||
|
List<EntityId> referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); |
||||
|
for (EntityId referencedEntityId : referencedEntityIds) { |
||||
|
EntityType entityType = referencedEntityId.getEntityType(); |
||||
|
switch (entityType) { |
||||
|
case TENANT, CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); |
||||
|
default -> |
||||
|
throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.exception; |
||||
|
|
||||
|
public class CalculatedFieldStateException extends RuntimeException { |
||||
|
|
||||
|
public CalculatedFieldStateException(String message) { |
||||
|
super(message); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,72 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.exception.CalculatedFieldStateException; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
||||
|
|
||||
|
import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; |
||||
|
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
||||
|
|
||||
|
public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { |
||||
|
|
||||
|
@Autowired |
||||
|
private ActorSystemContext actorSystemContext; |
||||
|
|
||||
|
protected PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer; |
||||
|
|
||||
|
@Override |
||||
|
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer) { |
||||
|
this.eventConsumer = eventConsumer; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { |
||||
|
if (state.isSizeExceedsLimit()) { |
||||
|
throw new CalculatedFieldStateException("State size exceeds the maximum allowed limit. The state will not be persisted to RocksDB."); |
||||
|
} |
||||
|
doPersist(stateId, toProto(stateId, state), callback); |
||||
|
} |
||||
|
|
||||
|
protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback); |
||||
|
|
||||
|
@Override |
||||
|
public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
||||
|
doRemove(stateId, callback); |
||||
|
} |
||||
|
|
||||
|
protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); |
||||
|
|
||||
|
protected void processRestoredState(CalculatedFieldStateProto stateMsg) { |
||||
|
var id = fromProto(stateMsg.getId()); |
||||
|
var state = fromProto(stateMsg); |
||||
|
processRestoredState(id, state); |
||||
|
} |
||||
|
|
||||
|
protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { |
||||
|
actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state)); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,45 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
public interface CalculatedFieldCache { |
||||
|
|
||||
|
CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); |
||||
|
|
||||
|
List<CalculatedField> getCalculatedFieldsByEntityId(EntityId entityId); |
||||
|
|
||||
|
List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId); |
||||
|
|
||||
|
CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); |
||||
|
|
||||
|
List<CalculatedFieldCtx> getCalculatedFieldCtxsByEntityId(EntityId entityId); |
||||
|
|
||||
|
void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); |
||||
|
|
||||
|
void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); |
||||
|
|
||||
|
void evict(CalculatedFieldId calculatedFieldId); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,19 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
public interface CalculatedFieldInitService { |
||||
|
} |
||||
@ -0,0 +1,43 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.Argument; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
||||
|
|
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
public interface CalculatedFieldProcessingService { |
||||
|
|
||||
|
ListenableFuture<CalculatedFieldState> fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); |
||||
|
|
||||
|
Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments); |
||||
|
|
||||
|
void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List<CalculatedFieldId> cfIds, TbCallback callback); |
||||
|
|
||||
|
void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,44 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import com.google.common.util.concurrent.FutureCallback; |
||||
|
import org.thingsboard.rule.engine.api.AttributesDeleteRequest; |
||||
|
import org.thingsboard.rule.engine.api.AttributesSaveRequest; |
||||
|
import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; |
||||
|
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; |
||||
|
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; |
||||
|
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQueueService { |
||||
|
|
||||
|
/** |
||||
|
* Filter CFs based on the request entity. Push to the queue if any matching CF exist; |
||||
|
* |
||||
|
* @param request - telemetry save request; |
||||
|
* @param callback |
||||
|
*/ |
||||
|
void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback<Void> callback); |
||||
|
|
||||
|
void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback); |
||||
|
|
||||
|
void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback); |
||||
|
|
||||
|
void pushRequestToQueue(TimeseriesDeleteRequest request, List<String> result, FutureCallback<Void> callback); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,30 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.AttributeScope; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
||||
|
|
||||
|
@Data |
||||
|
public final class CalculatedFieldResult { |
||||
|
|
||||
|
private final OutputType type; |
||||
|
private final AttributeScope scope; |
||||
|
private final JsonNode result; |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,41 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.exception.CalculatedFieldStateException; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
||||
|
|
||||
|
import java.util.Set; |
||||
|
|
||||
|
public interface CalculatedFieldStateService { |
||||
|
|
||||
|
void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer); |
||||
|
|
||||
|
void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; |
||||
|
|
||||
|
void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); |
||||
|
|
||||
|
void restore(Set<TopicPartitionInfo> partitions); |
||||
|
|
||||
|
void stop(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,47 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import jakarta.annotation.PostConstruct; |
||||
|
import jakarta.annotation.PreDestroy; |
||||
|
import org.rocksdb.Options; |
||||
|
import org.rocksdb.WriteOptions; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
import org.thingsboard.server.edqs.util.TbRocksDb; |
||||
|
|
||||
|
@Component |
||||
|
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") |
||||
|
public class CfRocksDb extends TbRocksDb { |
||||
|
|
||||
|
public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) { |
||||
|
super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); |
||||
|
} |
||||
|
|
||||
|
@PostConstruct |
||||
|
@Override |
||||
|
public void init() { |
||||
|
super.init(); |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
@Override |
||||
|
public void close() { |
||||
|
super.close(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,187 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import lombok.Getter; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.script.api.tbel.TbelInvokeService; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.page.PageDataIterable; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; |
||||
|
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
||||
|
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
||||
|
import org.thingsboard.server.queue.util.AfterStartUp; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
|
||||
|
import java.util.Collections; |
||||
|
import java.util.List; |
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.ConcurrentMap; |
||||
|
import java.util.concurrent.CopyOnWriteArrayList; |
||||
|
import java.util.concurrent.locks.Lock; |
||||
|
import java.util.concurrent.locks.ReentrantLock; |
||||
|
|
||||
|
@Service |
||||
|
@Slf4j |
||||
|
@RequiredArgsConstructor |
||||
|
public class DefaultCalculatedFieldCache implements CalculatedFieldCache { |
||||
|
|
||||
|
private static final Integer UNKNOWN_PARTITION = -1; |
||||
|
|
||||
|
private final Lock calculatedFieldFetchLock = new ReentrantLock(); |
||||
|
|
||||
|
private final CalculatedFieldService calculatedFieldService; |
||||
|
private final TbelInvokeService tbelInvokeService; |
||||
|
private final ActorSystemContext actorSystemContext; |
||||
|
private final ApiLimitService apiLimitService; |
||||
|
|
||||
|
private final ConcurrentMap<CalculatedFieldId, CalculatedField> calculatedFields = new ConcurrentHashMap<>(); |
||||
|
private final ConcurrentMap<EntityId, List<CalculatedField>> entityIdCalculatedFields = new ConcurrentHashMap<>(); |
||||
|
private final ConcurrentMap<CalculatedFieldId, List<CalculatedFieldLink>> calculatedFieldLinks = new ConcurrentHashMap<>(); |
||||
|
private final ConcurrentMap<EntityId, List<CalculatedFieldLink>> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); |
||||
|
private final ConcurrentMap<CalculatedFieldId, CalculatedFieldCtx> calculatedFieldsCtx = new ConcurrentHashMap<>(); |
||||
|
|
||||
|
@Value("${calculatedField.initFetchPackSize:50000}") |
||||
|
@Getter |
||||
|
private int initFetchPackSize; |
||||
|
|
||||
|
@AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) |
||||
|
public void init() { |
||||
|
//TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService)
|
||||
|
PageDataIterable<CalculatedField> cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); |
||||
|
cfs.forEach(cf -> { |
||||
|
calculatedFields.putIfAbsent(cf.getId(), cf); |
||||
|
actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); |
||||
|
}); |
||||
|
calculatedFields.values().forEach(cf -> { |
||||
|
entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); |
||||
|
}); |
||||
|
PageDataIterable<CalculatedFieldLink> cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); |
||||
|
cfls.forEach(link -> { |
||||
|
calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link); |
||||
|
actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); |
||||
|
}); |
||||
|
calculatedFieldLinks.values().stream() |
||||
|
.flatMap(List::stream) |
||||
|
.forEach(link -> |
||||
|
entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { |
||||
|
return calculatedFields.get(calculatedFieldId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<CalculatedField> getCalculatedFieldsByEntityId(EntityId entityId) { |
||||
|
return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(EntityId entityId) { |
||||
|
return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { |
||||
|
CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); |
||||
|
if (ctx == null) { |
||||
|
calculatedFieldFetchLock.lock(); |
||||
|
try { |
||||
|
ctx = calculatedFieldsCtx.get(calculatedFieldId); |
||||
|
if (ctx == null) { |
||||
|
CalculatedField calculatedField = getCalculatedField(calculatedFieldId); |
||||
|
if (calculatedField != null) { |
||||
|
ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); |
||||
|
calculatedFieldsCtx.put(calculatedFieldId, ctx); |
||||
|
log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); |
||||
|
} |
||||
|
} |
||||
|
} finally { |
||||
|
calculatedFieldFetchLock.unlock(); |
||||
|
} |
||||
|
} |
||||
|
log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); |
||||
|
return ctx; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<CalculatedFieldCtx> getCalculatedFieldCtxsByEntityId(EntityId entityId) { |
||||
|
if (entityId == null) { |
||||
|
return Collections.emptyList(); |
||||
|
} |
||||
|
return getCalculatedFieldsByEntityId(entityId).stream() |
||||
|
.map(cf -> getCalculatedFieldCtx(cf.getId())) |
||||
|
.toList(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { |
||||
|
calculatedFieldFetchLock.lock(); |
||||
|
try { |
||||
|
CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); |
||||
|
EntityId cfEntityId = calculatedField.getEntityId(); |
||||
|
|
||||
|
calculatedFields.put(calculatedFieldId, calculatedField); |
||||
|
|
||||
|
entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); |
||||
|
|
||||
|
CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); |
||||
|
calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); |
||||
|
|
||||
|
configuration.getReferencedEntities().stream() |
||||
|
.filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) |
||||
|
.forEach(referencedEntityId -> { |
||||
|
entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) |
||||
|
.add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); |
||||
|
}); |
||||
|
} finally { |
||||
|
calculatedFieldFetchLock.unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { |
||||
|
evict(calculatedFieldId); |
||||
|
addCalculatedField(tenantId, calculatedFieldId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void evict(CalculatedFieldId calculatedFieldId) { |
||||
|
CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); |
||||
|
log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); |
||||
|
calculatedFieldLinks.remove(calculatedFieldId); |
||||
|
log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); |
||||
|
entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); |
||||
|
log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); |
||||
|
calculatedFieldsCtx.remove(calculatedFieldId); |
||||
|
log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); |
||||
|
entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); |
||||
|
log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,59 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import lombok.Getter; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.common.data.ProfileEntityIdInfo; |
||||
|
import org.thingsboard.server.common.data.page.PageDataIterable; |
||||
|
import org.thingsboard.server.dao.asset.AssetService; |
||||
|
import org.thingsboard.server.dao.device.DeviceService; |
||||
|
import org.thingsboard.server.queue.util.AfterStartUp; |
||||
|
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
||||
|
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; |
||||
|
|
||||
|
@Slf4j |
||||
|
@Service |
||||
|
@TbRuleEngineComponent |
||||
|
@RequiredArgsConstructor |
||||
|
public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { |
||||
|
|
||||
|
private final CalculatedFieldEntityProfileCache entityProfileCache; |
||||
|
private final AssetService assetService; |
||||
|
private final DeviceService deviceService; |
||||
|
|
||||
|
@Value("${calculated_fields.init_fetch_pack_size:50000}") |
||||
|
@Getter |
||||
|
private int initFetchPackSize; |
||||
|
|
||||
|
@AfterStartUp(order = AfterStartUp.CF_READ_PROFILE_ENTITIES_SERVICE) |
||||
|
public void initCalculatedFieldDefinitions() { |
||||
|
PageDataIterable<ProfileEntityIdInfo> deviceIdInfos = new PageDataIterable<>(deviceService::findProfileEntityIdInfos, initFetchPackSize); |
||||
|
for (ProfileEntityIdInfo idInfo : deviceIdInfos) { |
||||
|
log.trace("Processing device record: {}", idInfo); |
||||
|
entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); |
||||
|
} |
||||
|
PageDataIterable<ProfileEntityIdInfo> assetIdInfos = new PageDataIterable<>(assetService::findProfileEntityIdInfos, initFetchPackSize); |
||||
|
for (ProfileEntityIdInfo idInfo : assetIdInfos) { |
||||
|
log.trace("Processing asset record: {}", idInfo); |
||||
|
entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,325 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import com.google.common.util.concurrent.Futures; |
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import com.google.common.util.concurrent.ListeningExecutorService; |
||||
|
import com.google.common.util.concurrent.MoreExecutors; |
||||
|
import jakarta.annotation.PostConstruct; |
||||
|
import jakarta.annotation.PreDestroy; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.commons.lang3.math.NumberUtils; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.common.util.ThingsBoardExecutors; |
||||
|
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
||||
|
import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; |
||||
|
import org.thingsboard.server.cluster.TbClusterService; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.StringUtils; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.Argument; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.OutputType; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.kv.Aggregation; |
||||
|
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; |
||||
|
import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.BooleanDataEntry; |
||||
|
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
||||
|
import org.thingsboard.server.common.data.kv.KvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; |
||||
|
import org.thingsboard.server.common.data.kv.StringDataEntry; |
||||
|
import org.thingsboard.server.common.data.kv.TsKvEntry; |
||||
|
import org.thingsboard.server.common.data.msg.TbMsgType; |
||||
|
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; |
||||
|
import org.thingsboard.server.common.msg.TbMsg; |
||||
|
import org.thingsboard.server.common.msg.TbMsgMetaData; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.dao.attributes.AttributesService; |
||||
|
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
||||
|
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; |
||||
|
import org.thingsboard.server.queue.TbQueueCallback; |
||||
|
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
||||
|
import org.thingsboard.server.queue.discovery.PartitionService; |
||||
|
import org.thingsboard.server.queue.discovery.QueueKey; |
||||
|
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.Map.Entry; |
||||
|
import java.util.Optional; |
||||
|
import java.util.UUID; |
||||
|
import java.util.concurrent.ExecutionException; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
import static org.thingsboard.server.common.data.DataConstants.SCOPE; |
||||
|
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
||||
|
|
||||
|
@TbRuleEngineComponent |
||||
|
@Service |
||||
|
@Slf4j |
||||
|
@RequiredArgsConstructor |
||||
|
public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { |
||||
|
|
||||
|
private final AttributesService attributesService; |
||||
|
private final TimeseriesService timeseriesService; |
||||
|
private final TbClusterService clusterService; |
||||
|
private final ApiLimitService apiLimitService; |
||||
|
private final PartitionService partitionService; |
||||
|
|
||||
|
private ListeningExecutorService calculatedFieldCallbackExecutor; |
||||
|
|
||||
|
@PostConstruct |
||||
|
public void init() { |
||||
|
calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( |
||||
|
Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
public void stop() { |
||||
|
if (calculatedFieldCallbackExecutor != null) { |
||||
|
calculatedFieldCallbackExecutor.shutdownNow(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ListenableFuture<CalculatedFieldState> fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { |
||||
|
Map<String, ListenableFuture<ArgumentEntry>> argFutures = new HashMap<>(); |
||||
|
for (var entry : ctx.getArguments().entrySet()) { |
||||
|
var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; |
||||
|
var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); |
||||
|
argFutures.put(entry.getKey(), argValueFuture); |
||||
|
} |
||||
|
return Futures.whenAllComplete(argFutures.values()).call(() -> { |
||||
|
var result = createStateByType(ctx); |
||||
|
result.updateState(ctx, argFutures.entrySet().stream() |
||||
|
.collect(Collectors.toMap( |
||||
|
Entry::getKey, // Keep the key as is
|
||||
|
entry -> { |
||||
|
try { |
||||
|
// Resolve the future to get the value
|
||||
|
return entry.getValue().get(); |
||||
|
} catch (ExecutionException | InterruptedException e) { |
||||
|
throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); |
||||
|
} |
||||
|
} |
||||
|
))); |
||||
|
return result; |
||||
|
}, calculatedFieldCallbackExecutor); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public Map<String, ArgumentEntry> fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map<String, Argument> arguments) { |
||||
|
Map<String, ListenableFuture<ArgumentEntry>> argFutures = new HashMap<>(); |
||||
|
for (var entry : arguments.entrySet()) { |
||||
|
var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; |
||||
|
var argValueFuture = fetchKvEntry(tenantId, argEntityId, entry.getValue()); |
||||
|
argFutures.put(entry.getKey(), argValueFuture); |
||||
|
} |
||||
|
return argFutures.entrySet().stream() |
||||
|
.collect(Collectors.toMap( |
||||
|
Entry::getKey, // Keep the key as is
|
||||
|
entry -> { |
||||
|
try { |
||||
|
// Resolve the future to get the value
|
||||
|
return entry.getValue().get(); |
||||
|
} catch (ExecutionException | InterruptedException e) { |
||||
|
throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); |
||||
|
} |
||||
|
} |
||||
|
)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List<CalculatedFieldId> cfIds, TbCallback callback) { |
||||
|
try { |
||||
|
OutputType type = calculatedFieldResult.getType(); |
||||
|
TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; |
||||
|
TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; |
||||
|
TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); |
||||
|
clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { |
||||
|
@Override |
||||
|
public void onSuccess(TbQueueMsgMetadata metadata) { |
||||
|
callback.onSuccess(); |
||||
|
log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
callback.onFailure(t); |
||||
|
} |
||||
|
}); |
||||
|
} catch (Exception e) { |
||||
|
log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); |
||||
|
callback.onFailure(e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List<CalculatedFieldEntityCtxId> linkedCalculatedFields, TbCallback callback) { |
||||
|
Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts = new HashMap<>(); |
||||
|
List<CalculatedFieldEntityCtxId> broadcasts = new ArrayList<>(); |
||||
|
for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { |
||||
|
var linkEntityId = link.entityId(); |
||||
|
var linkEntityType = linkEntityId.getEntityType(); |
||||
|
// Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P.
|
||||
|
boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); |
||||
|
if (broadcast) { |
||||
|
broadcasts.add(link); |
||||
|
} else { |
||||
|
TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); |
||||
|
unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); |
||||
|
} |
||||
|
} |
||||
|
MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); |
||||
|
if (!broadcasts.isEmpty()) { |
||||
|
broadcast(broadcasts, msg, linkCallback); |
||||
|
} else { |
||||
|
linkCallback.onSuccess(); |
||||
|
} |
||||
|
if (!unicasts.isEmpty()) { |
||||
|
unicast(unicasts, msg, linkCallback); |
||||
|
} else { |
||||
|
linkCallback.onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void unicast(Map<TopicPartitionInfo, List<CalculatedFieldEntityCtxId>> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { |
||||
|
TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); |
||||
|
unicasts.forEach((topicPartitionInfo, ctxIds) -> { |
||||
|
CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); |
||||
|
clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), |
||||
|
ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
private void broadcast(List<CalculatedFieldEntityCtxId> broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { |
||||
|
TbQueueCallback callback = new TbCallbackWrapper(mainCallback); |
||||
|
CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); |
||||
|
clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); |
||||
|
} |
||||
|
|
||||
|
private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List<CalculatedFieldEntityCtxId> links) { |
||||
|
Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); |
||||
|
builder.setMsg(telemetryProto); |
||||
|
for (CalculatedFieldEntityCtxId link : links) { |
||||
|
builder.addLinks(toProto(link)); |
||||
|
} |
||||
|
return builder.build(); |
||||
|
} |
||||
|
|
||||
|
private ListenableFuture<ArgumentEntry> fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { |
||||
|
return switch (argument.getRefEntityKey().getType()) { |
||||
|
case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); |
||||
|
case ATTRIBUTE -> transformSingleValueArgument( |
||||
|
Futures.transform( |
||||
|
attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), |
||||
|
result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), |
||||
|
calculatedFieldCallbackExecutor) |
||||
|
); |
||||
|
case TS_LATEST -> transformSingleValueArgument( |
||||
|
Futures.transform( |
||||
|
timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), |
||||
|
result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), |
||||
|
calculatedFieldCallbackExecutor)); |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
private ListenableFuture<ArgumentEntry> transformSingleValueArgument(ListenableFuture<Optional<? extends KvEntry>> kvEntryFuture) { |
||||
|
return Futures.transform(kvEntryFuture, kvEntry -> { |
||||
|
if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { |
||||
|
return ArgumentEntry.createSingleValueArgument(kvEntry.get()); |
||||
|
} else { |
||||
|
return new SingleValueArgumentEntry(); |
||||
|
} |
||||
|
}, calculatedFieldCallbackExecutor); |
||||
|
} |
||||
|
|
||||
|
private ListenableFuture<ArgumentEntry> fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { |
||||
|
long currentTime = System.currentTimeMillis(); |
||||
|
long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); |
||||
|
long startTs = currentTime - timeWindow; |
||||
|
long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); |
||||
|
int argumentLimit = argument.getLimit(); |
||||
|
int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argument.getLimit(); |
||||
|
|
||||
|
ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); |
||||
|
ListenableFuture<List<TsKvEntry>> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); |
||||
|
|
||||
|
return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? new TsRollingArgumentEntry(limit, timeWindow) : ArgumentEntry.createTsRollingArgument(tsRolling, limit, timeWindow), calculatedFieldCallbackExecutor); |
||||
|
} |
||||
|
|
||||
|
private KvEntry createDefaultKvEntry(Argument argument) { |
||||
|
String key = argument.getRefEntityKey().getKey(); |
||||
|
String defaultValue = argument.getDefaultValue(); |
||||
|
if (StringUtils.isBlank(defaultValue)) { |
||||
|
return new StringDataEntry(key, null); |
||||
|
} |
||||
|
if (NumberUtils.isParsable(defaultValue)) { |
||||
|
return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); |
||||
|
} |
||||
|
if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { |
||||
|
return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); |
||||
|
} |
||||
|
return new StringDataEntry(key, defaultValue); |
||||
|
} |
||||
|
|
||||
|
private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { |
||||
|
return switch (ctx.getCfType()) { |
||||
|
case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); |
||||
|
case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
private static class TbCallbackWrapper implements TbQueueCallback { |
||||
|
private final TbCallback callback; |
||||
|
|
||||
|
public TbCallbackWrapper(TbCallback callback) { |
||||
|
this.callback = callback; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onSuccess(TbQueueMsgMetadata metadata) { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
callback.onFailure(t); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,275 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf; |
||||
|
|
||||
|
import com.google.common.util.concurrent.FutureCallback; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.rule.engine.api.AttributesDeleteRequest; |
||||
|
import org.thingsboard.rule.engine.api.AttributesSaveRequest; |
||||
|
import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; |
||||
|
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; |
||||
|
import org.thingsboard.server.cluster.TbClusterService; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldLink; |
||||
|
import org.thingsboard.server.common.data.id.AssetId; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.DeviceId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; |
||||
|
import org.thingsboard.server.common.data.kv.TsKvEntry; |
||||
|
import org.thingsboard.server.common.data.msg.TbMsgType; |
||||
|
import org.thingsboard.server.common.util.ProtoUtils; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
||||
|
import org.thingsboard.server.queue.TbQueueCallback; |
||||
|
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; |
||||
|
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
||||
|
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
||||
|
|
||||
|
import java.util.Collections; |
||||
|
import java.util.EnumSet; |
||||
|
import java.util.List; |
||||
|
import java.util.Set; |
||||
|
import java.util.UUID; |
||||
|
import java.util.function.Predicate; |
||||
|
import java.util.function.Supplier; |
||||
|
|
||||
|
import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; |
||||
|
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; |
||||
|
|
||||
|
@Service |
||||
|
@Slf4j |
||||
|
@RequiredArgsConstructor |
||||
|
public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { |
||||
|
|
||||
|
public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { |
||||
|
@Override |
||||
|
public void onSuccess(TbQueueMsgMetadata metadata) { |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
} |
||||
|
}; |
||||
|
|
||||
|
private final TbAssetProfileCache assetProfileCache; |
||||
|
private final TbDeviceProfileCache deviceProfileCache; |
||||
|
private final CalculatedFieldCache calculatedFieldCache; |
||||
|
private final TbClusterService clusterService; |
||||
|
|
||||
|
private static final Set<EntityType> supportedReferencedEntities = EnumSet.of( |
||||
|
EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT |
||||
|
); |
||||
|
|
||||
|
@Override |
||||
|
public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback<Void> callback) { |
||||
|
var tenantId = request.getTenantId(); |
||||
|
var entityId = request.getEntityId(); |
||||
|
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), |
||||
|
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback<Void> callback) { |
||||
|
pushRequestToQueue(request, null, callback); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void pushRequestToQueue(AttributesSaveRequest request, List<Long> result, FutureCallback<Void> callback) { |
||||
|
var tenantId = request.getTenantId(); |
||||
|
var entityId = request.getEntityId(); |
||||
|
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), |
||||
|
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void pushRequestToQueue(AttributesSaveRequest request, FutureCallback<Void> callback) { |
||||
|
pushRequestToQueue(request, null, callback); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void pushRequestToQueue(AttributesDeleteRequest request, List<String> result, FutureCallback<Void> callback) { |
||||
|
var tenantId = request.getTenantId(); |
||||
|
var entityId = request.getEntityId(); |
||||
|
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), |
||||
|
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void pushRequestToQueue(TimeseriesDeleteRequest request, List<String> result, FutureCallback<Void> callback) { |
||||
|
var tenantId = request.getTenantId(); |
||||
|
var entityId = request.getEntityId(); |
||||
|
|
||||
|
checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), |
||||
|
() -> toCalculatedFieldTelemetryMsgProto(request, result), callback); |
||||
|
} |
||||
|
|
||||
|
private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, |
||||
|
Predicate<CalculatedFieldCtx> mainEntityFilter, Predicate<CalculatedFieldCtx> linkedEntityFilter, |
||||
|
Supplier<ToCalculatedFieldMsg> msg, FutureCallback<Void> callback) { |
||||
|
boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); |
||||
|
if (send) { |
||||
|
clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); |
||||
|
} else { |
||||
|
if (callback != null) { |
||||
|
callback.onSuccess(null); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate<CalculatedFieldCtx> filter, Predicate<CalculatedFieldCtx> linkedEntityFilter) { |
||||
|
boolean send = false; |
||||
|
if (supportedReferencedEntities.contains(entityId.getEntityType())) { |
||||
|
send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); |
||||
|
if (!send) { |
||||
|
send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); |
||||
|
} |
||||
|
if (!send) { |
||||
|
send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() |
||||
|
.map(CalculatedFieldLink::getCalculatedFieldId) |
||||
|
.map(calculatedFieldCache::getCalculatedFieldCtx) |
||||
|
.anyMatch(linkedEntityFilter); |
||||
|
} |
||||
|
} |
||||
|
return send; |
||||
|
} |
||||
|
|
||||
|
private EntityId getProfileId(TenantId tenantId, EntityId entityId) { |
||||
|
return switch (entityId.getEntityType()) { |
||||
|
case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); |
||||
|
case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); |
||||
|
default -> null; |
||||
|
}; |
||||
|
} |
||||
|
|
||||
|
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { |
||||
|
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); |
||||
|
|
||||
|
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); |
||||
|
|
||||
|
List<TsKvEntry> entries = request.getEntries(); |
||||
|
List<Long> versions = result != null ? result.getVersions() : Collections.emptyList(); |
||||
|
|
||||
|
for (int i = 0; i < entries.size(); i++) { |
||||
|
TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); |
||||
|
if (result != null) { |
||||
|
tsProtoBuilder.setVersion(versions.get(i)); |
||||
|
} |
||||
|
telemetryMsg.addTsData(tsProtoBuilder.build()); |
||||
|
} |
||||
|
|
||||
|
msg.setTelemetryMsg(telemetryMsg.build()); |
||||
|
return msg.build(); |
||||
|
} |
||||
|
|
||||
|
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List<Long> versions) { |
||||
|
ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); |
||||
|
|
||||
|
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); |
||||
|
telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); |
||||
|
List<AttributeKvEntry> entries = request.getEntries(); |
||||
|
for (int i = 0; i < entries.size(); i++) { |
||||
|
AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder(); |
||||
|
if (versions != null) { |
||||
|
attrProtoBuilder.setVersion(versions.get(i)); |
||||
|
} |
||||
|
telemetryMsg.addAttrData(attrProtoBuilder.build()); |
||||
|
} |
||||
|
msg.setTelemetryMsg(telemetryMsg.build()); |
||||
|
|
||||
|
return msg.build(); |
||||
|
} |
||||
|
|
||||
|
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesDeleteRequest request, List<String> removedKeys) { |
||||
|
CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) |
||||
|
.setScope(AttributeScopeProto.valueOf(request.getScope().name())) |
||||
|
.addAllRemovedAttrKeys(removedKeys).build(); |
||||
|
return ToCalculatedFieldMsg.newBuilder() |
||||
|
.setTelemetryMsg(telemetryMsg) |
||||
|
.build(); |
||||
|
} |
||||
|
|
||||
|
private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesDeleteRequest request, List<String> removedKeys) { |
||||
|
CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) |
||||
|
.addAllRemovedTsKeys(removedKeys).build(); |
||||
|
return ToCalculatedFieldMsg.newBuilder() |
||||
|
.setTelemetryMsg(telemetryMsg) |
||||
|
.build(); |
||||
|
} |
||||
|
|
||||
|
private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List<CalculatedFieldId> calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { |
||||
|
CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); |
||||
|
|
||||
|
telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); |
||||
|
telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); |
||||
|
|
||||
|
telemetryMsg.setEntityType(entityId.getEntityType().name()); |
||||
|
telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); |
||||
|
telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); |
||||
|
|
||||
|
if (calculatedFieldIds != null) { |
||||
|
for (CalculatedFieldId cfId : calculatedFieldIds) { |
||||
|
telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (tbMsgId != null) { |
||||
|
telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); |
||||
|
telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); |
||||
|
} |
||||
|
|
||||
|
if (tbMsgType != null) { |
||||
|
telemetryMsg.setTbMsgType(tbMsgType.name()); |
||||
|
} |
||||
|
|
||||
|
return telemetryMsg; |
||||
|
} |
||||
|
|
||||
|
private static TbQueueCallback wrap(FutureCallback<Void> callback) { |
||||
|
if (callback != null) { |
||||
|
return new FutureCallbackWrapper(callback); |
||||
|
} else { |
||||
|
return DUMMY_TB_QUEUE_CALLBACK; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static class FutureCallbackWrapper implements TbQueueCallback { |
||||
|
private final FutureCallback<Void> callback; |
||||
|
|
||||
|
public FutureCallbackWrapper(FutureCallback<Void> callback) { |
||||
|
this.callback = callback; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onSuccess(TbQueueMsgMetadata metadata) { |
||||
|
callback.onSuccess(null); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
callback.onFailure(t); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.cache; |
||||
|
|
||||
|
import org.springframework.context.ApplicationListener; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
||||
|
|
||||
|
import java.util.Collection; |
||||
|
|
||||
|
public interface CalculatedFieldEntityProfileCache extends ApplicationListener<PartitionChangeEvent> { |
||||
|
|
||||
|
void add(TenantId tenantId, EntityId profileId, EntityId entityId); |
||||
|
|
||||
|
void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); |
||||
|
|
||||
|
void evict(TenantId tenantId, EntityId entityId); |
||||
|
|
||||
|
Collection<EntityId> getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); |
||||
|
|
||||
|
int getEntityIdPartition(TenantId tenantId, EntityId entityId); |
||||
|
} |
||||
@ -0,0 +1,93 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.cache; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.queue.discovery.PartitionService; |
||||
|
import org.thingsboard.server.queue.discovery.QueueKey; |
||||
|
import org.thingsboard.server.queue.discovery.TbApplicationEventListener; |
||||
|
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
||||
|
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
||||
|
|
||||
|
import java.util.Collection; |
||||
|
import java.util.Collections; |
||||
|
import java.util.List; |
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.ConcurrentMap; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
@TbRuleEngineComponent |
||||
|
@Service |
||||
|
@Slf4j |
||||
|
@RequiredArgsConstructor |
||||
|
//TODO ashvayka: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor;
|
||||
|
public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener<PartitionChangeEvent> implements CalculatedFieldEntityProfileCache { |
||||
|
|
||||
|
private static final Integer UNKNOWN = 0; |
||||
|
private final ConcurrentMap<TenantId, TenantEntityProfileCache> tenantCache = new ConcurrentHashMap<>(); |
||||
|
private final PartitionService partitionService; |
||||
|
private volatile List<Integer> myPartitions = Collections.emptyList(); |
||||
|
|
||||
|
@Override |
||||
|
protected void onTbApplicationEvent(PartitionChangeEvent event) { |
||||
|
myPartitions = event.getCfPartitions().stream() |
||||
|
.filter(TopicPartitionInfo::isMyPartition) |
||||
|
.map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); |
||||
|
//Naive approach that need to be improved.
|
||||
|
tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { |
||||
|
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
||||
|
var partition = tpi.getPartition().orElse(UNKNOWN); |
||||
|
tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) |
||||
|
.add(profileId, entityId, partition, tpi.isMyPartition()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { |
||||
|
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
||||
|
var partition = tpi.getPartition().orElse(UNKNOWN); |
||||
|
var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); |
||||
|
//TODO: make this method atomic;
|
||||
|
cache.remove(oldProfileId, entityId); |
||||
|
cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void evict(TenantId tenantId, EntityId entityId) { |
||||
|
var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); |
||||
|
cache.removeEntityId(entityId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public Collection<EntityId> getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { |
||||
|
return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { |
||||
|
var tpi = partitionService.resolve(QueueKey.CF, entityId); |
||||
|
return tpi.getPartition().orElse(UNKNOWN); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,122 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.cache; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Collection; |
||||
|
import java.util.Collections; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.HashSet; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.Set; |
||||
|
import java.util.concurrent.locks.ReadWriteLock; |
||||
|
import java.util.concurrent.locks.ReentrantReadWriteLock; |
||||
|
|
||||
|
public class TenantEntityProfileCache { |
||||
|
|
||||
|
private final ReadWriteLock lock = new ReentrantReadWriteLock(); |
||||
|
private final Map<Integer, Map<EntityId, Set<EntityId>>> allEntities = new HashMap<>(); |
||||
|
private final Map<EntityId, Set<EntityId>> myEntities = new HashMap<>(); |
||||
|
|
||||
|
public void setMyPartitions(List<Integer> myPartitions) { |
||||
|
lock.writeLock().lock(); |
||||
|
try { |
||||
|
myEntities.clear(); |
||||
|
myPartitions.forEach(partitionId -> { |
||||
|
var map = allEntities.get(partitionId); |
||||
|
if (map != null) { |
||||
|
map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); |
||||
|
} |
||||
|
}); |
||||
|
} finally { |
||||
|
lock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void removeProfileId(EntityId profileId) { |
||||
|
lock.writeLock().lock(); |
||||
|
try { |
||||
|
// Remove from allEntities
|
||||
|
allEntities.values().forEach(map -> map.remove(profileId)); |
||||
|
// Remove from myEntities
|
||||
|
myEntities.remove(profileId); |
||||
|
} finally { |
||||
|
lock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void removeEntityId(EntityId entityId) { |
||||
|
lock.writeLock().lock(); |
||||
|
try { |
||||
|
// Remove from allEntities
|
||||
|
allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); |
||||
|
// Remove from myEntities
|
||||
|
myEntities.values().forEach(set -> set.remove(entityId)); |
||||
|
} finally { |
||||
|
lock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void remove(EntityId profileId, EntityId entityId) { |
||||
|
lock.writeLock().lock(); |
||||
|
try { |
||||
|
// Remove from allEntities
|
||||
|
allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); |
||||
|
// Remove from myEntities
|
||||
|
removeSafely(myEntities, profileId, entityId); |
||||
|
} finally { |
||||
|
lock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { |
||||
|
lock.writeLock().lock(); |
||||
|
try { |
||||
|
if(EntityType.DEVICE.equals(profileId.getEntityType())){ |
||||
|
throw new RuntimeException("WTF?"); |
||||
|
} |
||||
|
if (mine) { |
||||
|
myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); |
||||
|
} |
||||
|
allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); |
||||
|
} finally { |
||||
|
lock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public Collection<EntityId> getMyEntityIdsByProfileId(EntityId profileId) { |
||||
|
lock.readLock().lock(); |
||||
|
try { |
||||
|
var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); |
||||
|
List<EntityId> result = new ArrayList<>(entities.size()); |
||||
|
result.addAll(entities); |
||||
|
return result; |
||||
|
} finally { |
||||
|
lock.readLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void removeSafely(Map<EntityId, Set<EntityId>> map, EntityId profileId, EntityId entityId) { |
||||
|
var set = map.get(profileId); |
||||
|
if (set != null) { |
||||
|
set.remove(entityId); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,34 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; |
||||
|
|
||||
|
@Data |
||||
|
@NoArgsConstructor |
||||
|
public class CalculatedFieldEntityCtx { |
||||
|
|
||||
|
private CalculatedFieldEntityCtxId id; |
||||
|
private CalculatedFieldState state; |
||||
|
|
||||
|
public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { |
||||
|
this.id = id; |
||||
|
this.state = state; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,28 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { |
||||
|
|
||||
|
public String toKey() { |
||||
|
return cfId + "_" + entityId; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,61 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonIgnore; |
||||
|
import com.fasterxml.jackson.annotation.JsonSubTypes; |
||||
|
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfArg; |
||||
|
import org.thingsboard.server.common.data.kv.KvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.TsKvEntry; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
@JsonTypeInfo( |
||||
|
use = JsonTypeInfo.Id.NAME, |
||||
|
include = JsonTypeInfo.As.PROPERTY, |
||||
|
property = "type" |
||||
|
) |
||||
|
@JsonSubTypes({ |
||||
|
@JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), |
||||
|
@JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") |
||||
|
}) |
||||
|
public interface ArgumentEntry { |
||||
|
|
||||
|
@JsonIgnore |
||||
|
ArgumentEntryType getType(); |
||||
|
|
||||
|
Object getValue(); |
||||
|
|
||||
|
boolean updateEntry(ArgumentEntry entry); |
||||
|
|
||||
|
boolean isEmpty(); |
||||
|
|
||||
|
TbelCfArg toTbelCfArg(); |
||||
|
|
||||
|
boolean isForceResetPrevious(); |
||||
|
|
||||
|
void setForceResetPrevious(boolean forceResetPrevious); |
||||
|
|
||||
|
static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { |
||||
|
return new SingleValueArgumentEntry(kvEntry); |
||||
|
} |
||||
|
|
||||
|
static ArgumentEntry createTsRollingArgument(List<TsKvEntry> kvEntries, int limit, long timeWindow) { |
||||
|
return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,20 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
public enum ArgumentEntryType { |
||||
|
SINGLE_VALUE, TS_ROLLING |
||||
|
} |
||||
@ -0,0 +1,103 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
import org.thingsboard.server.utils.CalculatedFieldUtils; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; |
||||
|
|
||||
|
@Data |
||||
|
@AllArgsConstructor |
||||
|
public abstract class BaseCalculatedFieldState implements CalculatedFieldState { |
||||
|
|
||||
|
protected List<String> requiredArguments; |
||||
|
protected Map<String, ArgumentEntry> arguments; |
||||
|
protected boolean sizeExceedsLimit; |
||||
|
|
||||
|
public BaseCalculatedFieldState(List<String> requiredArguments) { |
||||
|
this.requiredArguments = requiredArguments; |
||||
|
this.arguments = new HashMap<>(); |
||||
|
} |
||||
|
|
||||
|
public BaseCalculatedFieldState() { |
||||
|
this(new ArrayList<>(), new HashMap<>(), false); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues) { |
||||
|
if (arguments == null) { |
||||
|
arguments = new HashMap<>(); |
||||
|
} |
||||
|
|
||||
|
boolean stateUpdated = false; |
||||
|
|
||||
|
for (Map.Entry<String, ArgumentEntry> entry : argumentValues.entrySet()) { |
||||
|
String key = entry.getKey(); |
||||
|
ArgumentEntry newEntry = entry.getValue(); |
||||
|
|
||||
|
checkArgumentSize(key, newEntry, ctx); |
||||
|
|
||||
|
ArgumentEntry existingEntry = arguments.get(key); |
||||
|
|
||||
|
if (existingEntry == null || newEntry.isForceResetPrevious()) { |
||||
|
validateNewEntry(newEntry); |
||||
|
arguments.put(key, newEntry); |
||||
|
stateUpdated = true; |
||||
|
} else { |
||||
|
stateUpdated = existingEntry.updateEntry(newEntry); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return stateUpdated; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean isReady() { |
||||
|
return arguments.keySet().containsAll(requiredArguments) && |
||||
|
arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize) { |
||||
|
if (!sizeExceedsLimit && maxStateSize > 0 && CalculatedFieldUtils.toProto(ctxId, this).getSerializedSize() > maxStateSize) { |
||||
|
arguments.clear(); |
||||
|
sizeExceedsLimit = true; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx) { |
||||
|
if (entry instanceof TsRollingArgumentEntry) { |
||||
|
return; |
||||
|
} |
||||
|
if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { |
||||
|
if (ctx.getMaxSingleValueArgumentSize() > 0 && toSingleValueArgumentProto(name, singleValueArgumentEntry).getSerializedSize() > ctx.getMaxSingleValueArgumentSize()) { |
||||
|
throw new IllegalArgumentException("Single value size exceeds the maximum allowed limit. The argument will not be used for calculation."); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
protected abstract void validateNewEntry(ArgumentEntry newEntry); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,282 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import net.objecthunter.exp4j.Expression; |
||||
|
import net.objecthunter.exp4j.ExpressionBuilder; |
||||
|
import org.mvel2.MVEL; |
||||
|
import org.thingsboard.script.api.tbel.TbelInvokeService; |
||||
|
import org.thingsboard.server.common.data.AttributeScope; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.Argument; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.ArgumentType; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.Output; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.TsKvEntry; |
||||
|
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; |
||||
|
import org.thingsboard.server.common.data.util.TbPair; |
||||
|
import org.thingsboard.server.common.util.ProtoUtils; |
||||
|
import org.thingsboard.server.dao.usagerecord.ApiLimitService; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
@Data |
||||
|
public class CalculatedFieldCtx { |
||||
|
|
||||
|
private CalculatedField calculatedField; |
||||
|
|
||||
|
private CalculatedFieldId cfId; |
||||
|
private TenantId tenantId; |
||||
|
private EntityId entityId; |
||||
|
private CalculatedFieldType cfType; |
||||
|
private final Map<String, Argument> arguments; |
||||
|
private final Map<ReferencedEntityKey, String> mainEntityArguments; |
||||
|
private final Map<EntityId, Map<ReferencedEntityKey, String>> linkedEntityArguments; |
||||
|
|
||||
|
private final Map<TbPair<EntityId, ReferencedEntityKey>, String> referencedEntityKeys; |
||||
|
private final List<String> argNames; |
||||
|
private Output output; |
||||
|
private String expression; |
||||
|
private TbelInvokeService tbelInvokeService; |
||||
|
private CalculatedFieldScriptEngine calculatedFieldScriptEngine; |
||||
|
private ThreadLocal<Expression> customExpression; |
||||
|
|
||||
|
private boolean initialized; |
||||
|
|
||||
|
private long maxDataPointsPerRollingArg; |
||||
|
private long maxStateSize; |
||||
|
private long maxSingleValueArgumentSize; |
||||
|
|
||||
|
public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { |
||||
|
this.calculatedField = calculatedField; |
||||
|
|
||||
|
this.cfId = calculatedField.getId(); |
||||
|
this.tenantId = calculatedField.getTenantId(); |
||||
|
this.entityId = calculatedField.getEntityId(); |
||||
|
this.cfType = calculatedField.getType(); |
||||
|
CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); |
||||
|
this.arguments = configuration.getArguments(); |
||||
|
this.mainEntityArguments = new HashMap<>(); |
||||
|
this.linkedEntityArguments = new HashMap<>(); |
||||
|
for (Map.Entry<String, Argument> entry : arguments.entrySet()) { |
||||
|
var refId = entry.getValue().getRefEntityId(); |
||||
|
var refKey = entry.getValue().getRefEntityKey(); |
||||
|
if (refId == null) { |
||||
|
mainEntityArguments.put(refKey, entry.getKey()); |
||||
|
} else { |
||||
|
linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); |
||||
|
} |
||||
|
} |
||||
|
this.referencedEntityKeys = arguments.entrySet().stream() |
||||
|
.collect(Collectors.toMap( |
||||
|
entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), |
||||
|
Map.Entry::getKey |
||||
|
)); |
||||
|
this.argNames = new ArrayList<>(arguments.keySet()); |
||||
|
this.output = configuration.getOutput(); |
||||
|
this.expression = configuration.getExpression(); |
||||
|
this.tbelInvokeService = tbelInvokeService; |
||||
|
|
||||
|
this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); |
||||
|
this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; |
||||
|
this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; |
||||
|
} |
||||
|
|
||||
|
public void init() { |
||||
|
if (CalculatedFieldType.SCRIPT.equals(cfType)) { |
||||
|
try { |
||||
|
this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); |
||||
|
initialized = true; |
||||
|
} catch (Exception e) { |
||||
|
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); |
||||
|
} |
||||
|
} else { |
||||
|
if (isValidExpression(expression)) { |
||||
|
this.customExpression = ThreadLocal.withInitial(() -> |
||||
|
new ExpressionBuilder(expression) |
||||
|
.implicitMultiplication(true) |
||||
|
.variables(this.arguments.keySet()) |
||||
|
.build() |
||||
|
); |
||||
|
initialized = true; |
||||
|
} else { |
||||
|
throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { |
||||
|
if (tbelInvokeService == null) { |
||||
|
throw new IllegalArgumentException("TBEL script engine is disabled!"); |
||||
|
} |
||||
|
|
||||
|
List<String> ctxAndArgNames = new ArrayList<>(argNames.size() + 1); |
||||
|
ctxAndArgNames.add("ctx"); |
||||
|
ctxAndArgNames.addAll(argNames); |
||||
|
return new CalculatedFieldTbelScriptEngine( |
||||
|
tenantId, |
||||
|
tbelInvokeService, |
||||
|
expression, |
||||
|
ctxAndArgNames.toArray(String[]::new) |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
private boolean isValidExpression(String expression) { |
||||
|
try { |
||||
|
MVEL.compileExpression(expression); |
||||
|
return true; |
||||
|
} catch (Exception e) { |
||||
|
return false; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public boolean matches(List<AttributeKvEntry> values, AttributeScope scope) { |
||||
|
return matchesAttributes(mainEntityArguments, values, scope); |
||||
|
} |
||||
|
|
||||
|
public boolean linkMatches(EntityId entityId, List<AttributeKvEntry> values, AttributeScope scope) { |
||||
|
var map = linkedEntityArguments.get(entityId); |
||||
|
return map != null && matchesAttributes(map, values, scope); |
||||
|
} |
||||
|
|
||||
|
public boolean matches(List<TsKvEntry> values) { |
||||
|
return matchesTimeSeries(mainEntityArguments, values); |
||||
|
} |
||||
|
|
||||
|
public boolean linkMatches(EntityId entityId, List<TsKvEntry> values) { |
||||
|
var map = linkedEntityArguments.get(entityId); |
||||
|
return map != null && matchesTimeSeries(map, values); |
||||
|
} |
||||
|
|
||||
|
private boolean matchesAttributes(Map<ReferencedEntityKey, String> argMap, List<AttributeKvEntry> values, AttributeScope scope) { |
||||
|
for (AttributeKvEntry attrKv : values) { |
||||
|
ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); |
||||
|
if (argMap.containsKey(attrKey)) { |
||||
|
return true; |
||||
|
} |
||||
|
} |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
private boolean matchesTimeSeries(Map<ReferencedEntityKey, String> argMap, List<TsKvEntry> values) { |
||||
|
for (TsKvEntry tsKv : values) { |
||||
|
ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); |
||||
|
if (argMap.containsKey(latestKey)) { |
||||
|
return true; |
||||
|
} |
||||
|
ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); |
||||
|
if (argMap.containsKey(rollingKey)) { |
||||
|
return true; |
||||
|
} |
||||
|
} |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
public boolean matchesKeys(List<String> keys, AttributeScope scope) { |
||||
|
return matchesAttributesKeys(mainEntityArguments, keys, scope); |
||||
|
} |
||||
|
|
||||
|
public boolean matchesKeys(List<String> keys) { |
||||
|
return matchesTimeSeriesKeys(mainEntityArguments, keys); |
||||
|
} |
||||
|
|
||||
|
private boolean matchesAttributesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys, AttributeScope scope) { |
||||
|
for (String key : keys) { |
||||
|
ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); |
||||
|
if (argMap.containsKey(attrKey)) { |
||||
|
return true; |
||||
|
} |
||||
|
} |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
private boolean matchesTimeSeriesKeys(Map<ReferencedEntityKey, String> argMap, List<String> keys) { |
||||
|
for (String key : keys) { |
||||
|
ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); |
||||
|
if (argMap.containsKey(latestKey)) { |
||||
|
return true; |
||||
|
} |
||||
|
ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); |
||||
|
if (argMap.containsKey(rollingKey)) { |
||||
|
return true; |
||||
|
} |
||||
|
} |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
public boolean linkMatchesAttrKeys(EntityId entityId, List<String> keys, AttributeScope scope) { |
||||
|
var map = linkedEntityArguments.get(entityId); |
||||
|
return map != null && matchesAttributesKeys(map, keys, scope); |
||||
|
} |
||||
|
|
||||
|
public boolean linkMatchesTsKeys(EntityId entityId, List<String> keys) { |
||||
|
var map = linkedEntityArguments.get(entityId); |
||||
|
return map != null && matchesTimeSeriesKeys(map, keys); |
||||
|
} |
||||
|
|
||||
|
public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { |
||||
|
if (!proto.getTsDataList().isEmpty()) { |
||||
|
List<TsKvEntry> updatedTelemetry = proto.getTsDataList().stream() |
||||
|
.map(ProtoUtils::fromProto) |
||||
|
.toList(); |
||||
|
return linkMatches(entityId, updatedTelemetry); |
||||
|
} else if (!proto.getAttrDataList().isEmpty()) { |
||||
|
AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); |
||||
|
List<AttributeKvEntry> updatedTelemetry = proto.getAttrDataList().stream() |
||||
|
.map(ProtoUtils::fromProto) |
||||
|
.toList(); |
||||
|
return linkMatches(entityId, updatedTelemetry, scope); |
||||
|
} else if (!proto.getRemovedTsKeysList().isEmpty()) { |
||||
|
return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); |
||||
|
} else { |
||||
|
return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { |
||||
|
return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); |
||||
|
} |
||||
|
|
||||
|
public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { |
||||
|
boolean expressionChanged = !expression.equals(other.expression); |
||||
|
boolean outputChanged = !output.equals(other.output); |
||||
|
return expressionChanged || outputChanged; |
||||
|
} |
||||
|
|
||||
|
public boolean hasStateChanges(CalculatedFieldCtx other) { |
||||
|
boolean typeChanged = !cfType.equals(other.cfType); |
||||
|
boolean argumentsChanged = !arguments.equals(other.arguments); |
||||
|
return typeChanged || argumentsChanged; |
||||
|
} |
||||
|
|
||||
|
public String getSizeExceedsLimitMessage() { |
||||
|
return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,29 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
|
||||
|
public interface CalculatedFieldScriptEngine { |
||||
|
|
||||
|
ListenableFuture<Object> executeScriptAsync(Object[] args); |
||||
|
|
||||
|
ListenableFuture<JsonNode> executeJsonAsync(Object[] args); |
||||
|
|
||||
|
void destroy(); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,65 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonIgnore; |
||||
|
import com.fasterxml.jackson.annotation.JsonSubTypes; |
||||
|
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
|
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
@JsonTypeInfo( |
||||
|
use = JsonTypeInfo.Id.NAME, |
||||
|
include = JsonTypeInfo.As.PROPERTY, |
||||
|
property = "type" |
||||
|
) |
||||
|
@JsonSubTypes({ |
||||
|
@JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), |
||||
|
@JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), |
||||
|
}) |
||||
|
public interface CalculatedFieldState { |
||||
|
|
||||
|
@JsonIgnore |
||||
|
CalculatedFieldType getType(); |
||||
|
|
||||
|
Map<String, ArgumentEntry> getArguments(); |
||||
|
|
||||
|
void setRequiredArguments(List<String> requiredArguments); |
||||
|
|
||||
|
boolean updateState(CalculatedFieldCtx ctx, Map<String, ArgumentEntry> argumentValues); |
||||
|
|
||||
|
ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx); |
||||
|
|
||||
|
@JsonIgnore |
||||
|
boolean isReady(); |
||||
|
|
||||
|
boolean isSizeExceedsLimit(); |
||||
|
|
||||
|
@JsonIgnore |
||||
|
default boolean isSizeOk() { |
||||
|
return !isSizeExceedsLimit(); |
||||
|
} |
||||
|
|
||||
|
void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); |
||||
|
|
||||
|
void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,82 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
import com.google.common.util.concurrent.Futures; |
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import com.google.common.util.concurrent.MoreExecutors; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.script.api.ScriptType; |
||||
|
import org.thingsboard.script.api.tbel.TbelInvokeService; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
|
||||
|
import javax.script.ScriptException; |
||||
|
import java.util.UUID; |
||||
|
import java.util.concurrent.ExecutionException; |
||||
|
|
||||
|
@Slf4j |
||||
|
public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { |
||||
|
|
||||
|
private final TbelInvokeService tbelInvokeService; |
||||
|
|
||||
|
private final UUID scriptId; |
||||
|
private final TenantId tenantId; |
||||
|
|
||||
|
public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { |
||||
|
this.tenantId = tenantId; |
||||
|
this.tbelInvokeService = tbelInvokeService; |
||||
|
try { |
||||
|
this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); |
||||
|
} catch (Exception e) { |
||||
|
Throwable t = e; |
||||
|
if (e instanceof ExecutionException) { |
||||
|
t = e.getCause(); |
||||
|
} |
||||
|
throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ListenableFuture<Object> executeScriptAsync(Object[] args) { |
||||
|
log.trace("Executing script async, args {}", args); |
||||
|
return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), |
||||
|
o -> { |
||||
|
try { |
||||
|
return Futures.immediateFuture(o); |
||||
|
} catch (Exception e) { |
||||
|
if (e.getCause() instanceof ScriptException) { |
||||
|
return Futures.immediateFailedFuture(e.getCause()); |
||||
|
} else if (e.getCause() instanceof RuntimeException) { |
||||
|
return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); |
||||
|
} else { |
||||
|
return Futures.immediateFailedFuture(new ScriptException(e)); |
||||
|
} |
||||
|
} |
||||
|
}, MoreExecutors.directExecutor()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ListenableFuture<JsonNode> executeJsonAsync(Object[] args) { |
||||
|
return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void destroy() { |
||||
|
tbelInvokeService.release(this.scriptId); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,157 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.EntityIdFactory; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
||||
|
import org.thingsboard.server.queue.TbQueueCallback; |
||||
|
import org.thingsboard.server.queue.TbQueueMsgHeaders; |
||||
|
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
||||
|
import org.thingsboard.server.queue.common.consumer.QueueStateService; |
||||
|
import org.thingsboard.server.queue.discovery.PartitionService; |
||||
|
import org.thingsboard.server.queue.discovery.QueueKey; |
||||
|
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; |
||||
|
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; |
||||
|
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
|
||||
|
import java.util.Set; |
||||
|
import java.util.concurrent.atomic.AtomicInteger; |
||||
|
|
||||
|
import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.*; |
||||
|
|
||||
|
@Service |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine') && '${queue.type:null}'=='kafka'") |
||||
|
public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { |
||||
|
|
||||
|
private final TbRuleEngineQueueFactory queueFactory; |
||||
|
private final PartitionService partitionService; |
||||
|
|
||||
|
@Value("${queue.calculated_fields.poll_interval:25}") |
||||
|
private long pollInterval; |
||||
|
|
||||
|
private PartitionedQueueConsumerManager<TbProtoQueueMsg<CalculatedFieldStateProto>> stateConsumer; |
||||
|
private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>> stateProducer; |
||||
|
private QueueStateService<TbProtoQueueMsg<ToCalculatedFieldMsg>, TbProtoQueueMsg<CalculatedFieldStateProto>> queueStateService; |
||||
|
|
||||
|
private final AtomicInteger counter = new AtomicInteger(); |
||||
|
|
||||
|
@Override |
||||
|
public void init(PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer) { |
||||
|
super.init(eventConsumer); |
||||
|
this.stateConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<CalculatedFieldStateProto>>create() |
||||
|
.queueKey(QueueKey.CF_STATES) |
||||
|
.topic(partitionService.getTopic(QueueKey.CF_STATES)) |
||||
|
.pollInterval(pollInterval) |
||||
|
.msgPackProcessor((msgs, consumer, config) -> { |
||||
|
for (TbProtoQueueMsg<CalculatedFieldStateProto> msg : msgs) { |
||||
|
try { |
||||
|
if (msg.getValue() != null) { |
||||
|
processRestoredState(msg.getValue()); |
||||
|
} else { |
||||
|
processRestoredState(getStateId(msg.getHeaders()), null); |
||||
|
} |
||||
|
} catch (Throwable t) { |
||||
|
log.error("Failed to process state message: {}", msg, t); |
||||
|
} |
||||
|
|
||||
|
int processedMsgCount = counter.incrementAndGet(); |
||||
|
if (processedMsgCount % 10000 == 0) { |
||||
|
log.info("Processed {} calculated field state msgs", processedMsgCount); |
||||
|
} |
||||
|
} |
||||
|
}) |
||||
|
.consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) |
||||
|
.consumerExecutor(eventConsumer.getConsumerExecutor()) |
||||
|
.scheduler(eventConsumer.getScheduler()) |
||||
|
.taskExecutor(eventConsumer.getTaskExecutor()) |
||||
|
.build(); |
||||
|
this.stateProducer = (TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>>) queueFactory.createCalculatedFieldStateProducer(); |
||||
|
this.queueStateService = new QueueStateService<>(); |
||||
|
this.queueStateService.init(stateConsumer, super.eventConsumer); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { |
||||
|
TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); |
||||
|
TbProtoQueueMsg<CalculatedFieldStateProto> msg = new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto); |
||||
|
if (stateMsgProto == null) { |
||||
|
putStateId(msg.getHeaders(), stateId); |
||||
|
} |
||||
|
stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { |
||||
|
@Override |
||||
|
public void onSuccess(TbQueueMsgMetadata metadata) { |
||||
|
if (callback != null) { |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onFailure(Throwable t) { |
||||
|
if (callback != null) { |
||||
|
callback.onFailure(t); |
||||
|
} |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
||||
|
doPersist(stateId, null, callback); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void restore(Set<TopicPartitionInfo> partitions) { |
||||
|
queueStateService.update(partitions); |
||||
|
} |
||||
|
|
||||
|
private void putStateId(TbQueueMsgHeaders headers, CalculatedFieldEntityCtxId stateId) { |
||||
|
headers.put("tenantId", uuidToBytes(stateId.tenantId().getId())); |
||||
|
headers.put("cfId", uuidToBytes(stateId.cfId().getId())); |
||||
|
headers.put("entityId", uuidToBytes(stateId.entityId().getId())); |
||||
|
headers.put("entityType", stringToBytes(stateId.entityId().getEntityType().name())); |
||||
|
} |
||||
|
|
||||
|
private CalculatedFieldEntityCtxId getStateId(TbQueueMsgHeaders headers) { |
||||
|
TenantId tenantId = TenantId.fromUUID(bytesToUuid(headers.get("tenantId"))); |
||||
|
CalculatedFieldId cfId = new CalculatedFieldId(bytesToUuid(headers.get("cfId"))); |
||||
|
EntityId entityId = EntityIdFactory.getByTypeAndUuid(bytesToString(headers.get("entityType")), bytesToUuid(headers.get("entityId"))); |
||||
|
return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void stop() { |
||||
|
stateConsumer.stop(); |
||||
|
stateConsumer.awaitStop(); |
||||
|
stateProducer.stop(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,73 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.google.protobuf.InvalidProtocolBufferException; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; |
||||
|
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; |
||||
|
import org.thingsboard.server.service.cf.CfRocksDb; |
||||
|
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; |
||||
|
|
||||
|
import java.util.Set; |
||||
|
|
||||
|
@Service |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") |
||||
|
public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { |
||||
|
|
||||
|
private final CfRocksDb cfRocksDb; |
||||
|
|
||||
|
private boolean initialized; |
||||
|
|
||||
|
@Override |
||||
|
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { |
||||
|
cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { |
||||
|
cfRocksDb.delete(stateId.toKey()); |
||||
|
callback.onSuccess(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void restore(Set<TopicPartitionInfo> partitions) { |
||||
|
if (!this.initialized) { |
||||
|
cfRocksDb.forEach((key, value) -> { |
||||
|
try { |
||||
|
processRestoredState(CalculatedFieldStateProto.parseFrom(value)); |
||||
|
} catch (InvalidProtocolBufferException e) { |
||||
|
log.error("[{}] Failed to process restored state", key, e); |
||||
|
} |
||||
|
}); |
||||
|
this.initialized = true; |
||||
|
} |
||||
|
eventConsumer.update(partitions); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void stop() { |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,83 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
import com.google.common.util.concurrent.Futures; |
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import com.google.common.util.concurrent.MoreExecutors; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfArg; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfCtx; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.Output; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.LinkedHashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
@Data |
||||
|
@Slf4j |
||||
|
@NoArgsConstructor |
||||
|
public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { |
||||
|
|
||||
|
public ScriptCalculatedFieldState(List<String> requiredArguments) { |
||||
|
super(requiredArguments); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public CalculatedFieldType getType() { |
||||
|
return CalculatedFieldType.SCRIPT; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void validateNewEntry(ArgumentEntry newEntry) { |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { |
||||
|
Map<String, TbelCfArg> arguments = new LinkedHashMap<>(); |
||||
|
List<Object> args = new ArrayList<>(ctx.getArgNames().size() + 1); |
||||
|
args.add(new Object()); // first element is a ctx, but we will set it later;
|
||||
|
for (String argName : ctx.getArgNames()) { |
||||
|
var arg = toTbelArgument(argName); |
||||
|
arguments.put(argName, arg); |
||||
|
if (arg instanceof TbelCfSingleValueArg svArg) { |
||||
|
args.add(svArg.getValue()); |
||||
|
} else { |
||||
|
args.add(arg); |
||||
|
} |
||||
|
} |
||||
|
args.set(0, new TbelCfCtx(arguments)); |
||||
|
ListenableFuture<JsonNode> resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray()); |
||||
|
Output output = ctx.getOutput(); |
||||
|
return Futures.transform(resultFuture, |
||||
|
result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), |
||||
|
MoreExecutors.directExecutor() |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
private TbelCfArg toTbelArgument(String key) { |
||||
|
return arguments.get(key).toTbelCfArg(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,83 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.google.common.util.concurrent.Futures; |
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.script.api.tbel.TbUtils; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedFieldType; |
||||
|
import org.thingsboard.server.common.data.cf.configuration.Output; |
||||
|
import org.thingsboard.server.common.data.kv.BasicKvEntry; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldResult; |
||||
|
|
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
@Data |
||||
|
@NoArgsConstructor |
||||
|
public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { |
||||
|
|
||||
|
public SimpleCalculatedFieldState(List<String> requiredArguments) { |
||||
|
super(requiredArguments); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public CalculatedFieldType getType() { |
||||
|
return CalculatedFieldType.SIMPLE; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void validateNewEntry(ArgumentEntry newEntry) { |
||||
|
if (newEntry instanceof TsRollingArgumentEntry) { |
||||
|
throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { |
||||
|
var expr = ctx.getCustomExpression().get(); |
||||
|
|
||||
|
for (Map.Entry<String, ArgumentEntry> entry : this.arguments.entrySet()) { |
||||
|
try { |
||||
|
BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); |
||||
|
expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); |
||||
|
} catch (NumberFormatException e) { |
||||
|
throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
double expressionResult = expr.evaluate(); |
||||
|
|
||||
|
Output output = ctx.getOutput(); |
||||
|
Object result; |
||||
|
Integer decimals = output.getDecimalsByDefault(); |
||||
|
if (decimals != null) { |
||||
|
if (decimals.equals(0)) { |
||||
|
result = TbUtils.toInt(expressionResult); |
||||
|
} else { |
||||
|
result = TbUtils.toFixed(expressionResult, decimals); |
||||
|
} |
||||
|
} else { |
||||
|
result = expressionResult; |
||||
|
} |
||||
|
|
||||
|
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result)))); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,115 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonIgnore; |
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfArg; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; |
||||
|
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.BasicKvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.KvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.TsKvEntry; |
||||
|
import org.thingsboard.server.common.util.ProtoUtils; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; |
||||
|
|
||||
|
@Data |
||||
|
@NoArgsConstructor |
||||
|
@AllArgsConstructor |
||||
|
public class SingleValueArgumentEntry implements ArgumentEntry { |
||||
|
|
||||
|
private long ts; |
||||
|
private BasicKvEntry kvEntryValue; |
||||
|
private Long version; |
||||
|
|
||||
|
private boolean forceResetPrevious; |
||||
|
|
||||
|
public SingleValueArgumentEntry(TsKvProto entry) { |
||||
|
this.ts = entry.getTs(); |
||||
|
if (entry.hasVersion()) { |
||||
|
this.version = entry.getVersion(); |
||||
|
} |
||||
|
this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); |
||||
|
} |
||||
|
|
||||
|
public SingleValueArgumentEntry(AttributeValueProto entry) { |
||||
|
this.ts = entry.getLastUpdateTs(); |
||||
|
if (entry.hasVersion()) { |
||||
|
this.version = entry.getVersion(); |
||||
|
} |
||||
|
this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); |
||||
|
} |
||||
|
|
||||
|
public SingleValueArgumentEntry(KvEntry entry) { |
||||
|
if (entry instanceof TsKvEntry tsKvEntry) { |
||||
|
this.ts = tsKvEntry.getTs(); |
||||
|
this.version = tsKvEntry.getVersion(); |
||||
|
} else if (entry instanceof AttributeKvEntry attributeKvEntry) { |
||||
|
this.ts = attributeKvEntry.getLastUpdateTs(); |
||||
|
this.version = attributeKvEntry.getVersion(); |
||||
|
} |
||||
|
this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); |
||||
|
} |
||||
|
|
||||
|
public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { |
||||
|
this.ts = ts; |
||||
|
this.kvEntryValue = kvEntryValue; |
||||
|
this.version = version; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ArgumentEntryType getType() { |
||||
|
return ArgumentEntryType.SINGLE_VALUE; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean isEmpty() { |
||||
|
return kvEntryValue == null; |
||||
|
} |
||||
|
|
||||
|
@JsonIgnore |
||||
|
public Object getValue() { |
||||
|
return isEmpty() ? null : kvEntryValue.getValue(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public TbelCfArg toTbelCfArg() { |
||||
|
return new TbelCfSingleValueArg(ts, kvEntryValue.getValue()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean updateEntry(ArgumentEntry entry) { |
||||
|
if (entry instanceof SingleValueArgumentEntry singleValueEntry) { |
||||
|
if (singleValueEntry.getTs() == this.ts) { |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
Long newVersion = singleValueEntry.getVersion(); |
||||
|
if (newVersion == null || this.version == null || newVersion > this.version) { |
||||
|
this.ts = singleValueEntry.getTs(); |
||||
|
this.version = newVersion; |
||||
|
this.kvEntryValue = singleValueEntry.getKvEntryValue(); |
||||
|
return true; |
||||
|
} |
||||
|
} else { |
||||
|
throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); |
||||
|
} |
||||
|
return false; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,146 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.cf.ctx.state; |
||||
|
|
||||
|
import com.fasterxml.jackson.annotation.JsonIgnore; |
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfArg; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; |
||||
|
import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; |
||||
|
import org.thingsboard.server.common.data.kv.KvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.TsKvEntry; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.TreeMap; |
||||
|
|
||||
|
@Data |
||||
|
@NoArgsConstructor |
||||
|
@AllArgsConstructor |
||||
|
@Slf4j |
||||
|
public class TsRollingArgumentEntry implements ArgumentEntry { |
||||
|
|
||||
|
private Integer limit; |
||||
|
private Long timeWindow; |
||||
|
private TreeMap<Long, Double> tsRecords = new TreeMap<>(); |
||||
|
|
||||
|
private boolean forceResetPrevious; |
||||
|
|
||||
|
public TsRollingArgumentEntry(List<TsKvEntry> kvEntries, int limit, long timeWindow) { |
||||
|
this.limit = limit; |
||||
|
this.timeWindow = timeWindow; |
||||
|
kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); |
||||
|
} |
||||
|
|
||||
|
public TsRollingArgumentEntry(TreeMap<Long, Double> tsRecords, int limit, long timeWindow) { |
||||
|
this.tsRecords = tsRecords; |
||||
|
this.limit = limit; |
||||
|
this.timeWindow = timeWindow; |
||||
|
} |
||||
|
|
||||
|
public TsRollingArgumentEntry(int limit, long timeWindow) { |
||||
|
this.tsRecords = new TreeMap<>(); |
||||
|
this.limit = limit; |
||||
|
this.timeWindow = timeWindow; |
||||
|
} |
||||
|
|
||||
|
public TsRollingArgumentEntry(Integer limit, Long timeWindow, TreeMap<Long, Double> tsRecords) { |
||||
|
this.limit = limit; |
||||
|
this.timeWindow = timeWindow; |
||||
|
this.tsRecords = tsRecords; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ArgumentEntryType getType() { |
||||
|
return ArgumentEntryType.TS_ROLLING; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean isEmpty() { |
||||
|
return tsRecords.isEmpty(); |
||||
|
} |
||||
|
|
||||
|
@JsonIgnore |
||||
|
@Override |
||||
|
public Object getValue() { |
||||
|
return tsRecords; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public TbelCfArg toTbelCfArg() { |
||||
|
List<TbelCfTsDoubleVal> values = new ArrayList<>(tsRecords.size()); |
||||
|
for (var e : tsRecords.entrySet()) { |
||||
|
values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); |
||||
|
} |
||||
|
return new TbelCfTsRollingArg(timeWindow, values); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean updateEntry(ArgumentEntry entry) { |
||||
|
if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { |
||||
|
updateTsRollingEntry(tsRollingEntry); |
||||
|
} else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { |
||||
|
updateSingleValueEntry(singleValueEntry); |
||||
|
} else { |
||||
|
throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); |
||||
|
} |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
private void updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { |
||||
|
for (Map.Entry<Long, Double> tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { |
||||
|
addTsRecord(tsRecordEntry.getKey(), tsRecordEntry.getValue()); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { |
||||
|
addTsRecord(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); |
||||
|
} |
||||
|
|
||||
|
private void addTsRecord(Long ts, KvEntry value) { |
||||
|
try { |
||||
|
switch (value.getDataType()) { |
||||
|
case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); |
||||
|
case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); |
||||
|
case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); |
||||
|
case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); |
||||
|
case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
tsRecords.put(ts, Double.NaN); |
||||
|
log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); |
||||
|
} finally { |
||||
|
cleanupExpiredRecords(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void addTsRecord(Long ts, double value) { |
||||
|
tsRecords.put(ts, value); |
||||
|
cleanupExpiredRecords(); |
||||
|
} |
||||
|
|
||||
|
private void cleanupExpiredRecords() { |
||||
|
if (tsRecords.size() > limit) { |
||||
|
tsRecords.pollFirstEntry(); |
||||
|
} |
||||
|
tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,82 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.edge.rpc.processor.rule; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.data.util.Pair; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.server.common.data.id.RuleChainId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.rule.RuleChain; |
||||
|
import org.thingsboard.server.common.data.rule.RuleChainMetaData; |
||||
|
import org.thingsboard.server.common.data.rule.RuleChainType; |
||||
|
import org.thingsboard.server.common.data.rule.RuleNode; |
||||
|
import org.thingsboard.server.dao.service.DataValidator; |
||||
|
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; |
||||
|
import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; |
||||
|
import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; |
||||
|
|
||||
|
import java.util.function.Function; |
||||
|
|
||||
|
@Slf4j |
||||
|
public class BaseRuleChainProcessor extends BaseEdgeProcessor { |
||||
|
|
||||
|
@Autowired |
||||
|
private DataValidator<RuleChain> ruleChainValidator; |
||||
|
|
||||
|
protected Pair<Boolean, Boolean> saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, RuleChainType ruleChainType) { |
||||
|
boolean created = false; |
||||
|
RuleChain ruleChainFromDb = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId); |
||||
|
if (ruleChainFromDb == null) { |
||||
|
created = true; |
||||
|
} |
||||
|
|
||||
|
RuleChain ruleChain = JacksonUtil.fromString(ruleChainUpdateMsg.getEntity(), RuleChain.class, true); |
||||
|
if (ruleChain == null) { |
||||
|
throw new RuntimeException("[{" + tenantId + "}] ruleChainUpdateMsg {" + ruleChainUpdateMsg + "} cannot be converted to rule chain"); |
||||
|
} |
||||
|
boolean isRoot = ruleChain.isRoot(); |
||||
|
if (RuleChainType.CORE.equals(ruleChainType)) { |
||||
|
ruleChain.setRoot(false); |
||||
|
} else { |
||||
|
ruleChain.setRoot(ruleChainFromDb == null ? false : ruleChainFromDb.isRoot()); |
||||
|
} |
||||
|
ruleChain.setType(ruleChainType); |
||||
|
|
||||
|
ruleChainValidator.validate(ruleChain, RuleChain::getTenantId); |
||||
|
if (created) { |
||||
|
ruleChain.setId(ruleChainId); |
||||
|
} |
||||
|
edgeCtx.getRuleChainService().saveRuleChain(ruleChain, true, false); |
||||
|
return Pair.of(created, isRoot); |
||||
|
} |
||||
|
|
||||
|
protected void saveOrUpdateRuleChainMetadata(TenantId tenantId, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) { |
||||
|
RuleChainMetaData ruleChainMetadata = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); |
||||
|
if (ruleChainMetadata == null) { |
||||
|
throw new RuntimeException("[{" + tenantId + "}] ruleChainMetadataUpdateMsg {" + ruleChainMetadataUpdateMsg + "} cannot be converted to rule chain metadata"); |
||||
|
} |
||||
|
if (!ruleChainMetadata.getNodes().isEmpty()) { |
||||
|
ruleChainMetadata.setVersion(null); |
||||
|
for (RuleNode ruleNode : ruleChainMetadata.getNodes()) { |
||||
|
ruleNode.setRuleChainId(null); |
||||
|
ruleNode.setId(null); |
||||
|
} |
||||
|
edgeCtx.getRuleChainService().saveRuleChainMetaData(tenantId, ruleChainMetadata, Function.identity(), true); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,117 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.edqs; |
||||
|
|
||||
|
import com.google.common.util.concurrent.Futures; |
||||
|
import com.google.common.util.concurrent.ListenableFuture; |
||||
|
import com.google.common.util.concurrent.MoreExecutors; |
||||
|
import jakarta.annotation.PostConstruct; |
||||
|
import jakarta.annotation.PreDestroy; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.server.common.data.edqs.query.EdqsRequest; |
||||
|
import org.thingsboard.server.common.data.edqs.query.EdqsResponse; |
||||
|
import org.thingsboard.server.common.data.id.CustomerId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.msg.edqs.EdqsApiService; |
||||
|
import org.thingsboard.server.edqs.state.EdqsPartitionService; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; |
||||
|
import org.thingsboard.server.queue.TbQueueRequestTemplate; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@Service |
||||
|
@Slf4j |
||||
|
@RequiredArgsConstructor |
||||
|
@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')") |
||||
|
public class DefaultEdqsApiService implements EdqsApiService { |
||||
|
|
||||
|
private final EdqsPartitionService edqsPartitionService; |
||||
|
private final EdqsClientQueueFactory queueFactory; |
||||
|
private TbQueueRequestTemplate<TbProtoQueueMsg<ToEdqsMsg>, TbProtoQueueMsg<FromEdqsMsg>> requestTemplate; |
||||
|
|
||||
|
@Value("${queue.edqs.api.auto_enable:true}") |
||||
|
private boolean autoEnable; |
||||
|
|
||||
|
private Boolean apiEnabled = null; |
||||
|
|
||||
|
@PostConstruct |
||||
|
private void init() { |
||||
|
requestTemplate = queueFactory.createEdqsRequestTemplate(); |
||||
|
requestTemplate.init(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ListenableFuture<EdqsResponse> processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { |
||||
|
var requestMsg = ToEdqsMsg.newBuilder() |
||||
|
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) |
||||
|
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) |
||||
|
.setTs(System.currentTimeMillis()) |
||||
|
.setRequestMsg(TransportProtos.EdqsRequestMsg.newBuilder() |
||||
|
.setValue(JacksonUtil.toString(request)) |
||||
|
.build()); |
||||
|
if (customerId != null && !customerId.isNullUid()) { |
||||
|
requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits()); |
||||
|
requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits()); |
||||
|
} |
||||
|
|
||||
|
Integer partition = edqsPartitionService.resolvePartition(tenantId); |
||||
|
ListenableFuture<TbProtoQueueMsg<FromEdqsMsg>> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition); |
||||
|
return Futures.transform(resultFuture, msg -> { |
||||
|
TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg(); |
||||
|
return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class); |
||||
|
}, MoreExecutors.directExecutor()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean isEnabled() { |
||||
|
return Boolean.TRUE.equals(apiEnabled); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void setEnabled(boolean enabled) { |
||||
|
if (enabled) { |
||||
|
log.info("Enabling EDQS API"); |
||||
|
} else { |
||||
|
log.info("Disabling EDQS API"); |
||||
|
} |
||||
|
apiEnabled = enabled; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean isSupported() { |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean isAutoEnable() { |
||||
|
return autoEnable; |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
private void stop() { |
||||
|
requestTemplate.stop(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,298 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.edqs; |
||||
|
|
||||
|
import com.google.protobuf.ByteString; |
||||
|
import jakarta.annotation.PostConstruct; |
||||
|
import jakarta.annotation.PreDestroy; |
||||
|
import lombok.AllArgsConstructor; |
||||
|
import lombok.Data; |
||||
|
import lombok.NoArgsConstructor; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.SneakyThrows; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
||||
|
import org.springframework.context.annotation.Lazy; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.common.util.ThingsBoardExecutors; |
||||
|
import org.thingsboard.server.cluster.TbClusterService; |
||||
|
import org.thingsboard.server.common.data.AttributeScope; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.ObjectType; |
||||
|
import org.thingsboard.server.common.data.edqs.EdqsEventType; |
||||
|
import org.thingsboard.server.common.data.edqs.EdqsObject; |
||||
|
import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; |
||||
|
import org.thingsboard.server.common.data.edqs.Entity; |
||||
|
import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; |
||||
|
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
||||
|
import org.thingsboard.server.common.data.kv.JsonDataEntry; |
||||
|
import org.thingsboard.server.common.data.kv.KvEntry; |
||||
|
import org.thingsboard.server.common.msg.edqs.EdqsApiService; |
||||
|
import org.thingsboard.server.common.msg.edqs.EdqsService; |
||||
|
import org.thingsboard.server.common.msg.queue.ServiceType; |
||||
|
import org.thingsboard.server.dao.attributes.AttributesService; |
||||
|
import org.thingsboard.server.edqs.processor.EdqsProducer; |
||||
|
import org.thingsboard.server.edqs.state.EdqsPartitionService; |
||||
|
import org.thingsboard.server.edqs.util.EdqsConverter; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; |
||||
|
import org.thingsboard.server.queue.discovery.HashPartitionService; |
||||
|
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; |
||||
|
import org.thingsboard.server.queue.discovery.TopicService; |
||||
|
import org.thingsboard.server.queue.edqs.EdqsQueue; |
||||
|
import org.thingsboard.server.queue.environment.DistributedLock; |
||||
|
import org.thingsboard.server.queue.environment.DistributedLockService; |
||||
|
import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; |
||||
|
import org.thingsboard.server.queue.util.AfterStartUp; |
||||
|
|
||||
|
import java.util.concurrent.ExecutorService; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
@Service |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") |
||||
|
public class DefaultEdqsService implements EdqsService { |
||||
|
|
||||
|
private final EdqsClientQueueFactory queueFactory; |
||||
|
private final EdqsConverter edqsConverter; |
||||
|
private final EdqsSyncService edqsSyncService; |
||||
|
private final EdqsApiService edqsApiService; |
||||
|
private final DistributedLockService distributedLockService; |
||||
|
private final AttributesService attributesService; |
||||
|
private final EdqsPartitionService edqsPartitionService; |
||||
|
private final TopicService topicService; |
||||
|
private final TbServiceInfoProvider serviceInfoProvider; |
||||
|
@Autowired @Lazy |
||||
|
private TbClusterService clusterService; |
||||
|
@Autowired @Lazy |
||||
|
private HashPartitionService hashPartitionService; |
||||
|
|
||||
|
private EdqsProducer eventsProducer; |
||||
|
private ExecutorService executor; |
||||
|
private DistributedLock syncLock; |
||||
|
|
||||
|
@PostConstruct |
||||
|
private void init() { |
||||
|
executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); |
||||
|
eventsProducer = EdqsProducer.builder() |
||||
|
.queue(EdqsQueue.EVENTS) |
||||
|
.partitionService(edqsPartitionService) |
||||
|
.topicService(topicService) |
||||
|
.producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) |
||||
|
.build(); |
||||
|
syncLock = distributedLockService.getLock("edqs_sync"); |
||||
|
} |
||||
|
|
||||
|
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) |
||||
|
public void onStartUp() { |
||||
|
if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) { |
||||
|
return; |
||||
|
} |
||||
|
executor.submit(() -> { |
||||
|
try { |
||||
|
EdqsSyncState syncState = getSyncState(); |
||||
|
if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { |
||||
|
if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { |
||||
|
processSystemRequest(ToCoreEdqsRequest.builder() |
||||
|
.syncRequest(new EdqsSyncRequest()) |
||||
|
.build()); |
||||
|
} |
||||
|
} else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) { |
||||
|
// only if topic/RocksDB is not empty and sync is finished
|
||||
|
edqsApiService.setEnabled(true); |
||||
|
} |
||||
|
} catch (Throwable e) { |
||||
|
log.error("Failed to start EDQS service", e); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void processSystemRequest(ToCoreEdqsRequest request) { |
||||
|
log.info("Processing system request {}", request); |
||||
|
if (request.getSyncRequest() != null) { |
||||
|
saveSyncState(EdqsSyncStatus.REQUESTED); |
||||
|
} |
||||
|
broadcast(request.toInternalMsg()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void processSystemMsg(ToCoreEdqsMsg msg) { |
||||
|
executor.submit(() -> { |
||||
|
log.info("Processing system msg {}", msg); |
||||
|
try { |
||||
|
if (msg.getApiEnabled() != null) { |
||||
|
edqsApiService.setEnabled(msg.getApiEnabled()); |
||||
|
} |
||||
|
|
||||
|
if (msg.getSyncRequest() != null) { |
||||
|
syncLock.lock(); |
||||
|
try { |
||||
|
EdqsSyncState syncState = getSyncState(); |
||||
|
if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) { |
||||
|
log.info("EDQS sync is already finished"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
saveSyncState(EdqsSyncStatus.STARTED); |
||||
|
edqsSyncService.sync(); |
||||
|
saveSyncState(EdqsSyncStatus.FINISHED); |
||||
|
|
||||
|
if (edqsApiService.isSupported()) |
||||
|
if (edqsApiService.isAutoEnable()) { |
||||
|
log.info("EDQS sync is finished, auto-enabling API"); |
||||
|
broadcast(ToCoreEdqsMsg.builder() |
||||
|
.apiEnabled(Boolean.TRUE) |
||||
|
.build()); |
||||
|
} else { |
||||
|
log.info("EDQS sync is finished, but leaving API disabled"); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
log.error("Failed to complete sync", e); |
||||
|
saveSyncState(EdqsSyncStatus.FAILED); |
||||
|
} finally { |
||||
|
syncLock.unlock(); |
||||
|
} |
||||
|
} |
||||
|
} catch (Throwable e) { |
||||
|
log.error("Failed to process msg {}", msg, e); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) { |
||||
|
EntityType entityType = entityId.getEntityType(); |
||||
|
ObjectType objectType = ObjectType.fromEntityType(entityType); |
||||
|
if (!isEdqsType(tenantId, objectType)) { |
||||
|
log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType); |
||||
|
return; |
||||
|
} |
||||
|
onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) { |
||||
|
processEvent(tenantId, objectType, EdqsEventType.UPDATED, object); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onDelete(TenantId tenantId, EntityId entityId) { |
||||
|
EntityType entityType = entityId.getEntityType(); |
||||
|
ObjectType objectType = ObjectType.fromEntityType(entityType); |
||||
|
if (!isEdqsType(tenantId, objectType)) { |
||||
|
log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType); |
||||
|
return; |
||||
|
} |
||||
|
onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) { |
||||
|
processEvent(tenantId, objectType, EdqsEventType.DELETED, object); |
||||
|
} |
||||
|
|
||||
|
protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) { |
||||
|
executor.submit(() -> { |
||||
|
try { |
||||
|
String key = object.key(); |
||||
|
Long version = object.version(); |
||||
|
EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder() |
||||
|
.setKey(key) |
||||
|
.setObjectType(objectType.name()) |
||||
|
.setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object))) |
||||
|
.setEventType(eventType.name()); |
||||
|
if (version != null) { |
||||
|
eventMsg.setVersion(version); |
||||
|
} |
||||
|
eventsProducer.send(tenantId, objectType, key, ToEdqsMsg.newBuilder() |
||||
|
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) |
||||
|
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) |
||||
|
.setTs(System.currentTimeMillis()) |
||||
|
.setEventMsg(eventMsg) |
||||
|
.build()); |
||||
|
} catch (Throwable e) { |
||||
|
log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
private boolean isEdqsType(TenantId tenantId, ObjectType objectType) { |
||||
|
if (objectType == null) { |
||||
|
return false; |
||||
|
} |
||||
|
if (!tenantId.isSysTenantId()) { |
||||
|
return ObjectType.edqsTypes.contains(objectType); |
||||
|
} else { |
||||
|
return ObjectType.edqsSystemTypes.contains(objectType); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void broadcast(ToCoreEdqsMsg msg) { |
||||
|
clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder() |
||||
|
.setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder() |
||||
|
.setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg)))) |
||||
|
.build()); |
||||
|
} |
||||
|
|
||||
|
@SneakyThrows |
||||
|
private EdqsSyncState getSyncState() { |
||||
|
EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS) |
||||
|
.flatMap(KvEntry::getJsonValue) |
||||
|
.map(value -> JacksonUtil.fromString(value, EdqsSyncState.class)) |
||||
|
.orElse(null); |
||||
|
log.info("EDQS sync state: {}", state); |
||||
|
return state; |
||||
|
} |
||||
|
|
||||
|
@SneakyThrows |
||||
|
private void saveSyncState(EdqsSyncStatus status) { |
||||
|
EdqsSyncState state = new EdqsSyncState(status); |
||||
|
log.info("New EDQS sync state: {}", state); |
||||
|
attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry( |
||||
|
new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)), |
||||
|
System.currentTimeMillis())).get(30, TimeUnit.SECONDS); |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
private void stop() { |
||||
|
executor.shutdown(); |
||||
|
eventsProducer.stop(); |
||||
|
} |
||||
|
|
||||
|
@Data |
||||
|
@AllArgsConstructor |
||||
|
@NoArgsConstructor |
||||
|
private static class EdqsSyncState { |
||||
|
private EdqsSyncStatus status; |
||||
|
} |
||||
|
|
||||
|
private enum EdqsSyncStatus { |
||||
|
REQUESTED, |
||||
|
STARTED, |
||||
|
FINISHED, |
||||
|
FAILED |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,61 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.edqs; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.transaction.event.TransactionalEventListener; |
||||
|
import org.thingsboard.server.common.data.ObjectType; |
||||
|
import org.thingsboard.server.common.data.audit.ActionType; |
||||
|
import org.thingsboard.server.common.msg.edqs.EdqsService; |
||||
|
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; |
||||
|
import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; |
||||
|
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; |
||||
|
|
||||
|
@Service |
||||
|
@RequiredArgsConstructor |
||||
|
@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") |
||||
|
public class EdqsListener { |
||||
|
|
||||
|
private final EdqsService edqsService; |
||||
|
|
||||
|
@TransactionalEventListener(fallbackExecution = true) |
||||
|
public void onUpdate(SaveEntityEvent<?> event) { |
||||
|
if (event.getEntityId() == null || event.getEntity() == null) { |
||||
|
return; |
||||
|
} |
||||
|
edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity()); |
||||
|
} |
||||
|
|
||||
|
@TransactionalEventListener(fallbackExecution = true) |
||||
|
public void onDelete(DeleteEntityEvent<?> event) { |
||||
|
if (event.getEntityId() == null) { |
||||
|
return; |
||||
|
} |
||||
|
edqsService.onDelete(event.getTenantId(), event.getEntityId()); |
||||
|
} |
||||
|
|
||||
|
@TransactionalEventListener(fallbackExecution = true) |
||||
|
public void handleEvent(RelationActionEvent relationEvent) { |
||||
|
if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { |
||||
|
edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); |
||||
|
} else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { |
||||
|
edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,284 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.edqs; |
||||
|
|
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.context.annotation.Lazy; |
||||
|
import org.thingsboard.server.common.data.AttributeScope; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.ObjectType; |
||||
|
import org.thingsboard.server.common.data.edqs.AttributeKv; |
||||
|
import org.thingsboard.server.common.data.edqs.EdqsEventType; |
||||
|
import org.thingsboard.server.common.data.edqs.EdqsObject; |
||||
|
import org.thingsboard.server.common.data.edqs.Entity; |
||||
|
import org.thingsboard.server.common.data.edqs.LatestTsKv; |
||||
|
import org.thingsboard.server.common.data.edqs.fields.EntityFields; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.EntityIdFactory; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.page.PageDataIterable; |
||||
|
import org.thingsboard.server.common.data.relation.RelationTypeGroup; |
||||
|
import org.thingsboard.server.dao.Dao; |
||||
|
import org.thingsboard.server.dao.attributes.AttributesDao; |
||||
|
import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; |
||||
|
import org.thingsboard.server.dao.entity.EntityDaoRegistry; |
||||
|
import org.thingsboard.server.dao.model.sql.AttributeKvEntity; |
||||
|
import org.thingsboard.server.dao.model.sql.RelationEntity; |
||||
|
import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; |
||||
|
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; |
||||
|
import org.thingsboard.server.dao.sql.relation.RelationRepository; |
||||
|
import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; |
||||
|
|
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.UUID; |
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.atomic.AtomicInteger; |
||||
|
|
||||
|
import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; |
||||
|
import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; |
||||
|
import static org.thingsboard.server.common.data.ObjectType.RELATION; |
||||
|
import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; |
||||
|
|
||||
|
@Slf4j |
||||
|
public abstract class EdqsSyncService { |
||||
|
|
||||
|
@Value("${queue.edqs.sync.entity_batch_size:10000}") |
||||
|
private int entityBatchSize; |
||||
|
@Value("${queue.edqs.sync.ts_batch_size:10000}") |
||||
|
private int tsBatchSize; |
||||
|
@Autowired |
||||
|
private EntityDaoRegistry entityDaoRegistry; |
||||
|
@Autowired |
||||
|
private AttributesDao attributesDao; |
||||
|
@Autowired |
||||
|
private KeyDictionaryDao keyDictionaryDao; |
||||
|
@Autowired |
||||
|
private RelationRepository relationRepository; |
||||
|
@Autowired |
||||
|
private TsKvLatestRepository tsKvLatestRepository; |
||||
|
@Autowired |
||||
|
@Lazy |
||||
|
private DefaultEdqsService edqsService; |
||||
|
|
||||
|
private final ConcurrentHashMap<UUID, EntityIdInfo> entityInfoMap = new ConcurrentHashMap<>(); |
||||
|
private final ConcurrentHashMap<Integer, String> keys = new ConcurrentHashMap<>(); |
||||
|
|
||||
|
private final Map<ObjectType, AtomicInteger> counters = new ConcurrentHashMap<>(); |
||||
|
|
||||
|
public abstract boolean isSyncNeeded(); |
||||
|
|
||||
|
public void sync() { |
||||
|
log.info("Synchronizing data to EDQS"); |
||||
|
long startTs = System.currentTimeMillis(); |
||||
|
counters.clear(); |
||||
|
|
||||
|
syncTenantEntities(); |
||||
|
syncRelations(); |
||||
|
loadKeyDictionary(); |
||||
|
syncAttributes(); |
||||
|
syncLatestTimeseries(); |
||||
|
|
||||
|
counters.clear(); |
||||
|
log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); |
||||
|
} |
||||
|
|
||||
|
private void process(TenantId tenantId, ObjectType type, EdqsObject object) { |
||||
|
AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger()); |
||||
|
if (counter.incrementAndGet() % 10000 == 0) { |
||||
|
log.info("Processed {} {} objects", counter.get(), type); |
||||
|
} |
||||
|
edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); |
||||
|
} |
||||
|
|
||||
|
private void syncTenantEntities() { |
||||
|
for (ObjectType type : edqsTenantTypes) { |
||||
|
log.info("Synchronizing {} entities to EDQS", type); |
||||
|
long ts = System.currentTimeMillis(); |
||||
|
EntityType entityType = type.toEntityType(); |
||||
|
Dao<?> dao = entityDaoRegistry.getDao(entityType); |
||||
|
UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
||||
|
while (true) { |
||||
|
var batch = dao.findNextBatch(lastId, entityBatchSize); |
||||
|
if (batch.isEmpty()) { |
||||
|
break; |
||||
|
} |
||||
|
for (EntityFields entityFields : batch) { |
||||
|
TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); |
||||
|
entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); |
||||
|
process(tenantId, type, new Entity(entityType, entityFields)); |
||||
|
} |
||||
|
EntityFields lastRecord = batch.get(batch.size() - 1); |
||||
|
lastId = lastRecord.getId(); |
||||
|
} |
||||
|
log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void syncRelations() { |
||||
|
log.info("Synchronizing relations to EDQS"); |
||||
|
long ts = System.currentTimeMillis(); |
||||
|
UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
||||
|
String lastFromEntityType = ""; |
||||
|
String lastRelationTypeGroup = ""; |
||||
|
String lastRelationType = ""; |
||||
|
UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
||||
|
String lastToEntityType = ""; |
||||
|
|
||||
|
while (true) { |
||||
|
List<RelationEntity> batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, |
||||
|
lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize); |
||||
|
if (batch.isEmpty()) { |
||||
|
break; |
||||
|
} |
||||
|
processRelationBatch(batch); |
||||
|
|
||||
|
RelationEntity lastRecord = batch.get(batch.size() - 1); |
||||
|
lastFromEntityId = lastRecord.getFromId(); |
||||
|
lastFromEntityType = lastRecord.getFromType(); |
||||
|
lastRelationTypeGroup = lastRecord.getRelationTypeGroup(); |
||||
|
lastRelationType = lastRecord.getRelationType(); |
||||
|
lastToEntityId = lastRecord.getToId(); |
||||
|
lastToEntityType = lastRecord.getToType(); |
||||
|
} |
||||
|
log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); |
||||
|
} |
||||
|
|
||||
|
private void processRelationBatch(List<RelationEntity> relations) { |
||||
|
for (RelationEntity relation : relations) { |
||||
|
if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) { |
||||
|
EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); |
||||
|
if (entityIdInfo != null) { |
||||
|
process(entityIdInfo.tenantId(), RELATION, relation.toData()); |
||||
|
} else { |
||||
|
log.info("Relation from id not found: {} ", relation); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void loadKeyDictionary() { |
||||
|
log.info("Loading key dictionary"); |
||||
|
long ts = System.currentTimeMillis(); |
||||
|
var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000); |
||||
|
for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) { |
||||
|
keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey()); |
||||
|
} |
||||
|
log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts)); |
||||
|
} |
||||
|
|
||||
|
private void syncAttributes() { |
||||
|
log.info("Synchronizing attributes to EDQS"); |
||||
|
long ts = System.currentTimeMillis(); |
||||
|
|
||||
|
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
||||
|
int lastAttributeType = Integer.MIN_VALUE; |
||||
|
int lastAttributeKey = Integer.MIN_VALUE; |
||||
|
|
||||
|
while (true) { |
||||
|
List<AttributeKvEntity> batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize); |
||||
|
if (batch.isEmpty()) { |
||||
|
break; |
||||
|
} |
||||
|
processAttributeBatch(batch); |
||||
|
|
||||
|
AttributeKvEntity lastRecord = batch.get(batch.size() - 1); |
||||
|
lastEntityId = lastRecord.getId().getEntityId(); |
||||
|
lastAttributeType = lastRecord.getId().getAttributeType(); |
||||
|
lastAttributeKey = lastRecord.getId().getAttributeKey(); |
||||
|
} |
||||
|
log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); |
||||
|
} |
||||
|
|
||||
|
private void processAttributeBatch(List<AttributeKvEntity> batch) { |
||||
|
for (AttributeKvEntity attribute : batch) { |
||||
|
attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); |
||||
|
UUID entityId = attribute.getId().getEntityId(); |
||||
|
EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); |
||||
|
if (entityIdInfo == null) { |
||||
|
log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId); |
||||
|
continue; |
||||
|
} |
||||
|
AttributeKv attributeKv = new AttributeKv( |
||||
|
EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId), |
||||
|
AttributeScope.valueOf(attribute.getId().getAttributeType()), |
||||
|
attribute.toData(), |
||||
|
attribute.getVersion()); |
||||
|
process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void syncLatestTimeseries() { |
||||
|
log.info("Synchronizing latest timeseries to EDQS"); |
||||
|
long ts = System.currentTimeMillis(); |
||||
|
UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); |
||||
|
int lastKey = Integer.MIN_VALUE; |
||||
|
|
||||
|
while (true) { |
||||
|
List<TsKvLatestEntity> batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize); |
||||
|
if (batch.isEmpty()) { |
||||
|
break; |
||||
|
} |
||||
|
processTsKvLatestBatch(batch); |
||||
|
|
||||
|
TsKvLatestEntity lastRecord = batch.get(batch.size() - 1); |
||||
|
lastEntityId = lastRecord.getEntityId(); |
||||
|
lastKey = lastRecord.getKey(); |
||||
|
} |
||||
|
log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); |
||||
|
} |
||||
|
|
||||
|
private void processTsKvLatestBatch(List<TsKvLatestEntity> tsKvLatestEntities) { |
||||
|
for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { |
||||
|
try { |
||||
|
String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); |
||||
|
if (strKey == null) { |
||||
|
log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey()); |
||||
|
continue; |
||||
|
} |
||||
|
tsKvLatestEntity.setStrKey(strKey); |
||||
|
UUID entityUuid = tsKvLatestEntity.getEntityId(); |
||||
|
EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid); |
||||
|
if (entityIdInfo != null) { |
||||
|
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid); |
||||
|
LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion()); |
||||
|
process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private String getStrKeyOrFetchFromDb(int key) { |
||||
|
String strKey = keys.get(key); |
||||
|
if (strKey != null) { |
||||
|
return strKey; |
||||
|
} else { |
||||
|
strKey = keyDictionaryDao.getKey(key); |
||||
|
if (strKey != null) { |
||||
|
keys.put(key, strKey); |
||||
|
} |
||||
|
} |
||||
|
return strKey; |
||||
|
} |
||||
|
|
||||
|
public record EntityIdInfo(EntityType entityType, TenantId tenantId) { |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,42 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.edqs; |
||||
|
|
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.queue.edqs.EdqsQueue; |
||||
|
import org.thingsboard.server.queue.kafka.TbKafkaAdmin; |
||||
|
import org.thingsboard.server.queue.kafka.TbKafkaSettings; |
||||
|
|
||||
|
import java.util.Collections; |
||||
|
|
||||
|
@Service |
||||
|
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") |
||||
|
public class KafkaEdqsSyncService extends EdqsSyncService { |
||||
|
|
||||
|
private final boolean syncNeeded; |
||||
|
|
||||
|
public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) { |
||||
|
TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); |
||||
|
this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean isSyncNeeded() { |
||||
|
return syncNeeded; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,35 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.edqs; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.edqs.util.EdqsRocksDb; |
||||
|
|
||||
|
@Service |
||||
|
@RequiredArgsConstructor |
||||
|
@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") |
||||
|
public class LocalEdqsSyncService extends EdqsSyncService { |
||||
|
|
||||
|
private final EdqsRocksDb db; |
||||
|
|
||||
|
@Override |
||||
|
public boolean isSyncNeeded() { |
||||
|
return db.isNew(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,106 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.entitiy.cf; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.transaction.annotation.Transactional; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.audit.ActionType; |
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.exception.ThingsboardException; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.page.PageData; |
||||
|
import org.thingsboard.server.common.data.page.PageLink; |
||||
|
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
||||
|
import org.thingsboard.server.queue.util.TbCoreComponent; |
||||
|
import org.thingsboard.server.service.entitiy.AbstractTbEntityService; |
||||
|
import org.thingsboard.server.service.security.model.SecurityUser; |
||||
|
|
||||
|
import java.util.Optional; |
||||
|
|
||||
|
@TbCoreComponent |
||||
|
@Service |
||||
|
@Slf4j |
||||
|
@RequiredArgsConstructor |
||||
|
public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { |
||||
|
|
||||
|
private final CalculatedFieldService calculatedFieldService; |
||||
|
|
||||
|
@Override |
||||
|
public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { |
||||
|
ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; |
||||
|
TenantId tenantId = calculatedField.getTenantId(); |
||||
|
try { |
||||
|
if (ActionType.UPDATED.equals(actionType)) { |
||||
|
CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); |
||||
|
checkForEntityChange(existingCf, calculatedField); |
||||
|
} |
||||
|
checkEntityExistence(tenantId, calculatedField.getEntityId()); |
||||
|
CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); |
||||
|
logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); |
||||
|
return savedCalculatedField; |
||||
|
} catch (ThingsboardException e) { |
||||
|
logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); |
||||
|
throw e; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { |
||||
|
return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public PageData<CalculatedField> findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { |
||||
|
TenantId tenantId = user.getTenantId(); |
||||
|
checkEntityExistence(tenantId, entityId); |
||||
|
return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
@Transactional |
||||
|
public void delete(CalculatedField calculatedField, SecurityUser user) { |
||||
|
ActionType actionType = ActionType.DELETED; |
||||
|
TenantId tenantId = calculatedField.getTenantId(); |
||||
|
CalculatedFieldId calculatedFieldId = calculatedField.getId(); |
||||
|
try { |
||||
|
calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); |
||||
|
logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); |
||||
|
} catch (Exception e) { |
||||
|
logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); |
||||
|
throw e; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { |
||||
|
if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { |
||||
|
throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void checkEntityExistence(TenantId tenantId, EntityId entityId) { |
||||
|
switch (entityId.getEntityType()) { |
||||
|
case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) |
||||
|
.orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); |
||||
|
default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.entitiy.cf; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.cf.CalculatedField; |
||||
|
import org.thingsboard.server.common.data.exception.ThingsboardException; |
||||
|
import org.thingsboard.server.common.data.id.CalculatedFieldId; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.page.PageData; |
||||
|
import org.thingsboard.server.common.data.page.PageLink; |
||||
|
import org.thingsboard.server.service.security.model.SecurityUser; |
||||
|
|
||||
|
public interface TbCalculatedFieldService { |
||||
|
|
||||
|
CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; |
||||
|
|
||||
|
CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); |
||||
|
|
||||
|
PageData<CalculatedField> findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); |
||||
|
|
||||
|
void delete(CalculatedField calculatedField, SecurityUser user); |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,43 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.housekeeper.processor; |
||||
|
|
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; |
||||
|
import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; |
||||
|
import org.thingsboard.server.dao.cf.CalculatedFieldService; |
||||
|
|
||||
|
@Component |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor<HousekeeperTask> { |
||||
|
|
||||
|
private final CalculatedFieldService calculatedFieldService; |
||||
|
|
||||
|
@Override |
||||
|
public void process(HousekeeperTask task) throws Exception { |
||||
|
int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); |
||||
|
log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public HousekeeperTaskType getTaskType() { |
||||
|
return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,269 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.queue; |
||||
|
|
||||
|
import com.google.common.util.concurrent.ListeningExecutorService; |
||||
|
import com.google.common.util.concurrent.MoreExecutors; |
||||
|
import jakarta.annotation.PostConstruct; |
||||
|
import jakarta.annotation.PreDestroy; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.context.ApplicationEventPublisher; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.common.util.ThingsBoardExecutors; |
||||
|
import org.thingsboard.server.actors.ActorSystemContext; |
||||
|
import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; |
||||
|
import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; |
||||
|
import org.thingsboard.server.common.data.id.EntityIdFactory; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.queue.QueueConfig; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; |
||||
|
import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; |
||||
|
import org.thingsboard.server.common.msg.queue.ServiceType; |
||||
|
import org.thingsboard.server.common.msg.queue.TbCallback; |
||||
|
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
||||
|
import org.thingsboard.server.dao.tenant.TbTenantProfileCache; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; |
||||
|
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; |
||||
|
import org.thingsboard.server.queue.TbQueueConsumer; |
||||
|
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
||||
|
import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; |
||||
|
import org.thingsboard.server.queue.discovery.PartitionService; |
||||
|
import org.thingsboard.server.queue.discovery.QueueKey; |
||||
|
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
||||
|
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; |
||||
|
import org.thingsboard.server.queue.util.TbRuleEngineComponent; |
||||
|
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldCache; |
||||
|
import org.thingsboard.server.service.cf.CalculatedFieldStateService; |
||||
|
import org.thingsboard.server.service.profile.TbAssetProfileCache; |
||||
|
import org.thingsboard.server.service.profile.TbDeviceProfileCache; |
||||
|
import org.thingsboard.server.service.queue.processing.AbstractConsumerService; |
||||
|
import org.thingsboard.server.service.queue.processing.IdMsgPair; |
||||
|
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; |
||||
|
|
||||
|
import java.util.List; |
||||
|
import java.util.Set; |
||||
|
import java.util.UUID; |
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.ConcurrentMap; |
||||
|
import java.util.concurrent.CountDownLatch; |
||||
|
import java.util.concurrent.Future; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
import static org.thingsboard.server.common.util.ProtoUtils.fromProto; |
||||
|
|
||||
|
@Service |
||||
|
@TbRuleEngineComponent |
||||
|
@Slf4j |
||||
|
public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService<ToCalculatedFieldNotificationMsg> implements TbCalculatedFieldConsumerService { |
||||
|
|
||||
|
@Value("${queue.calculated_fields.poll_interval:25}") |
||||
|
private long pollInterval; |
||||
|
@Value("${queue.calculated_fields.pack_processing_timeout:60000}") |
||||
|
private long packProcessingTimeout; |
||||
|
@Value("${queue.calculated_fields.pool_size:8}") |
||||
|
private int poolSize; |
||||
|
|
||||
|
private final TbRuleEngineQueueFactory queueFactory; |
||||
|
private final CalculatedFieldStateService stateService; |
||||
|
|
||||
|
private PartitionedQueueConsumerManager<TbProtoQueueMsg<ToCalculatedFieldMsg>> eventConsumer; |
||||
|
|
||||
|
public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, |
||||
|
ActorSystemContext actorContext, |
||||
|
TbDeviceProfileCache deviceProfileCache, |
||||
|
TbAssetProfileCache assetProfileCache, |
||||
|
TbTenantProfileCache tenantProfileCache, |
||||
|
TbApiUsageStateService apiUsageStateService, |
||||
|
PartitionService partitionService, |
||||
|
ApplicationEventPublisher eventPublisher, |
||||
|
JwtSettingsService jwtSettingsService, |
||||
|
CalculatedFieldCache calculatedFieldCache, |
||||
|
CalculatedFieldStateService stateService) { |
||||
|
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, |
||||
|
eventPublisher, jwtSettingsService); |
||||
|
this.queueFactory = tbQueueFactory; |
||||
|
this.stateService = stateService; |
||||
|
} |
||||
|
|
||||
|
@PostConstruct |
||||
|
public void init() { |
||||
|
super.init("tb-cf"); |
||||
|
|
||||
|
this.eventConsumer = PartitionedQueueConsumerManager.<TbProtoQueueMsg<ToCalculatedFieldMsg>>create() |
||||
|
.queueKey(QueueKey.CF) |
||||
|
.topic(partitionService.getTopic(QueueKey.CF)) |
||||
|
.pollInterval(pollInterval) |
||||
|
.msgPackProcessor(this::processMsgs) |
||||
|
.consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) |
||||
|
.consumerExecutor(consumersExecutor) |
||||
|
.scheduler(scheduler) |
||||
|
.taskExecutor(mgmtExecutor) |
||||
|
.build(); |
||||
|
stateService.init(eventConsumer); |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
public void destroy() { |
||||
|
super.destroy(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void startConsumers() { |
||||
|
super.startConsumers(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void onTbApplicationEvent(PartitionChangeEvent event) { |
||||
|
var partitions = event.getCfPartitions(); |
||||
|
try { |
||||
|
stateService.restore(partitions); |
||||
|
// eventConsumer's partitions will be updated by stateService
|
||||
|
|
||||
|
// Cleanup old entities after corresponding consumers are stopped.
|
||||
|
// Any periodic tasks need to check that the entity is still managed by the current server before processing.
|
||||
|
actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); |
||||
|
} catch (Throwable t) { |
||||
|
log.error("Failed to process partition change event: {}", event, t); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private boolean[] partitionsToBooleanIndexArray(Set<TopicPartitionInfo> partitions) { |
||||
|
boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; |
||||
|
for (var tpi : partitions) { |
||||
|
tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); |
||||
|
} |
||||
|
return myPartitions; |
||||
|
} |
||||
|
|
||||
|
private void processMsgs(List<TbProtoQueueMsg<ToCalculatedFieldMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldMsg>> consumer, QueueConfig config) throws Exception { |
||||
|
List<IdMsgPair<ToCalculatedFieldMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); |
||||
|
ConcurrentMap<UUID, TbProtoQueueMsg<ToCalculatedFieldMsg>> pendingMap = orderedMsgList.stream().collect( |
||||
|
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); |
||||
|
CountDownLatch processingTimeoutLatch = new CountDownLatch(1); |
||||
|
TbPackProcessingContext<TbProtoQueueMsg<ToCalculatedFieldMsg>> ctx = new TbPackProcessingContext<>( |
||||
|
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); |
||||
|
PendingMsgHolder<ToCalculatedFieldMsg> pendingMsgHolder = new PendingMsgHolder<>(); |
||||
|
Future<?> packSubmitFuture = consumersExecutor.submit(() -> { |
||||
|
orderedMsgList.forEach((element) -> { |
||||
|
UUID id = element.getUuid(); |
||||
|
TbProtoQueueMsg<ToCalculatedFieldMsg> msg = element.getMsg(); |
||||
|
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); |
||||
|
TbCallback callback = new TbPackCallback<>(id, ctx); |
||||
|
try { |
||||
|
ToCalculatedFieldMsg toCfMsg = msg.getValue(); |
||||
|
pendingMsgHolder.setMsg(toCfMsg); |
||||
|
if (toCfMsg.hasTelemetryMsg()) { |
||||
|
log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); |
||||
|
forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); |
||||
|
} else if (toCfMsg.hasLinkedTelemetryMsg()) { |
||||
|
forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); |
||||
|
} else if (toCfMsg.hasComponentLifecycleMsg()) { |
||||
|
log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); |
||||
|
forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); |
||||
|
} |
||||
|
} catch (Throwable e) { |
||||
|
log.warn("[{}] Failed to process message: {}", id, msg, e); |
||||
|
callback.onFailure(e); |
||||
|
} |
||||
|
}); |
||||
|
}); |
||||
|
if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { |
||||
|
if (!packSubmitFuture.isDone()) { |
||||
|
packSubmitFuture.cancel(true); |
||||
|
log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); |
||||
|
} |
||||
|
if (log.isDebugEnabled()) { |
||||
|
ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); |
||||
|
} |
||||
|
ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); |
||||
|
} |
||||
|
consumer.commit(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected ServiceType getServiceType() { |
||||
|
return ServiceType.TB_RULE_ENGINE; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected long getNotificationPollDuration() { |
||||
|
return pollInterval; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected long getNotificationPackProcessingTimeout() { |
||||
|
return packProcessingTimeout; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected int getMgmtThreadPoolSize() { |
||||
|
return Math.max(Runtime.getRuntime().availableProcessors(), 4); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldNotificationMsg>> createNotificationsConsumer() { |
||||
|
return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void handleNotification(UUID id, TbProtoQueueMsg<ToCalculatedFieldNotificationMsg> msg, TbCallback callback) { |
||||
|
ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); |
||||
|
if (toCfNotification.hasComponentLifecycleMsg()) { |
||||
|
// from upstream (maybe removed since we don't need to init state for each partition)
|
||||
|
log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); |
||||
|
forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); |
||||
|
} else if (toCfNotification.hasLinkedTelemetryMsg()) { |
||||
|
forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { |
||||
|
var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); |
||||
|
var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); |
||||
|
actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); |
||||
|
} |
||||
|
|
||||
|
private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { |
||||
|
var msg = linkedMsg.getMsg(); |
||||
|
var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); |
||||
|
var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); |
||||
|
actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); |
||||
|
} |
||||
|
|
||||
|
private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { |
||||
|
var msg = fromProto(proto); |
||||
|
actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); |
||||
|
} |
||||
|
|
||||
|
private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { |
||||
|
return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
protected void stopConsumers() { |
||||
|
super.stopConsumers(); |
||||
|
eventConsumer.stop(); |
||||
|
eventConsumer.awaitStop(); |
||||
|
stateService.stop(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.queue; |
||||
|
|
||||
|
import lombok.Getter; |
||||
|
import lombok.Setter; |
||||
|
|
||||
|
public class PendingMsgHolder<T> { |
||||
|
@Getter @Setter |
||||
|
private volatile T msg; |
||||
|
} |
||||
@ -0,0 +1,23 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2025 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.queue; |
||||
|
|
||||
|
import org.springframework.context.ApplicationListener; |
||||
|
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; |
||||
|
|
||||
|
public interface TbCalculatedFieldConsumerService extends ApplicationListener<PartitionChangeEvent> { |
||||
|
|
||||
|
} |
||||
@ -1,48 +0,0 @@ |
|||||
/** |
|
||||
* Copyright © 2016-2025 The Thingsboard Authors |
|
||||
* |
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||
* you may not use this file except in compliance with the License. |
|
||||
* You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
package org.thingsboard.server.service.queue.ruleengine; |
|
||||
|
|
||||
import lombok.AllArgsConstructor; |
|
||||
import lombok.Getter; |
|
||||
import lombok.ToString; |
|
||||
import org.thingsboard.server.common.data.queue.QueueConfig; |
|
||||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|
||||
|
|
||||
import java.util.Set; |
|
||||
|
|
||||
@Getter |
|
||||
@ToString |
|
||||
@AllArgsConstructor |
|
||||
public class TbQueueConsumerManagerTask { |
|
||||
|
|
||||
private final QueueEvent event; |
|
||||
private QueueConfig config; |
|
||||
private Set<TopicPartitionInfo> partitions; |
|
||||
private boolean drainQueue; |
|
||||
|
|
||||
public static TbQueueConsumerManagerTask delete(boolean drainQueue) { |
|
||||
return new TbQueueConsumerManagerTask(QueueEvent.DELETE, null, null, drainQueue); |
|
||||
} |
|
||||
|
|
||||
public static TbQueueConsumerManagerTask configUpdate(QueueConfig config) { |
|
||||
return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, config, null, false); |
|
||||
} |
|
||||
|
|
||||
public static TbQueueConsumerManagerTask partitionChange(Set<TopicPartitionInfo> partitions) { |
|
||||
return new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, null, partitions, false); |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue