Browse Source

Merge remote-tracking branch 'origin/master' into feature/notification-system

pull/7511/head
ViacheslavKlimov 4 years ago
parent
commit
a2facbd648
  1. 100
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  2. 23
      application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java
  3. 47
      application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java
  4. 10
      application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java
  5. 193
      application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
  6. 2
      application/src/main/resources/thingsboard.yml
  7. 2
      application/src/test/java/org/thingsboard/server/service/script/LocalJsInvokeServiceTest.java
  8. 219
      application/src/test/java/org/thingsboard/server/service/script/RemoteJsInvokeServiceTest.java
  9. 17
      common/cluster-api/src/main/proto/jsinvoke.proto
  10. 4
      common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java
  11. 26
      common/data/src/main/java/org/thingsboard/server/common/data/HasRuleEngineProfile.java
  12. 10
      common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java
  13. 3
      common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java
  14. 18
      common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java
  15. 45
      dao/src/main/java/org/thingsboard/server/dao/aspect/SqlDaoCallsAspect.java
  16. 4
      dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java
  17. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java
  18. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java
  19. 4
      dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java
  20. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java
  21. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java
  22. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java
  23. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java
  24. 5
      msa/js-executor/api/jsExecutor.models.ts
  25. 82
      msa/js-executor/api/jsInvokeMessageProcessor.ts
  26. 4
      msa/js-executor/api/utils.ts
  27. 4
      msa/js-executor/queue/awsSqsTemplate.ts
  28. 5
      msa/js-executor/queue/kafkaTemplate.ts
  29. 4
      msa/js-executor/queue/pubSubTemplate.ts
  30. 2
      msa/js-executor/queue/queue.models.ts
  31. 4
      msa/js-executor/queue/rabbitmqTemplate.ts
  32. 4
      msa/js-executor/queue/serviceBusTemplate.ts
  33. 7
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java
  34. 47
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java
  35. 10
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java
  36. 45
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/AttributesDeleteNodeCallback.java
  37. 44
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/AttributesUpdateNodeCallback.java
  38. 16
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java
  39. 73
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributes.java
  40. 38
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesConfiguration.java
  41. 125
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesTest.java

100
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -17,8 +17,11 @@ package org.thingsboard.server.actors.ruleChain;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.netty.channel.EventLoopGroup;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ListeningExecutor;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.RuleEngineAlarmService;
@ -39,6 +42,7 @@ import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasRuleEngineProfile;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.alarm.Alarm;
@ -52,6 +56,7 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleNode;
@ -88,6 +93,7 @@ import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
@ -330,58 +336,62 @@ class DefaultTbContext implements TbContext {
}
public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) {
RuleChainId ruleChainId = null;
String queueName = null;
DeviceProfile deviceProfile = null;
if (device.getDeviceProfileId() != null) {
DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().find(device.getDeviceProfileId());
if (deviceProfile == null) {
log.warn("[{}] Device profile is null!", device.getDeviceProfileId());
} else {
ruleChainId = deviceProfile.getDefaultRuleChainId();
queueName = deviceProfile.getDefaultQueueName();
}
deviceProfile = mainCtx.getDeviceProfileCache().find(device.getDeviceProfileId());
}
return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, queueName, ruleChainId);
return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, deviceProfile);
}
public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) {
RuleChainId ruleChainId = null;
String queueName = null;
AssetProfile assetProfile = null;
if (asset.getAssetProfileId() != null) {
AssetProfile assetProfile = mainCtx.getAssetProfileCache().find(asset.getAssetProfileId());
if (assetProfile == null) {
log.warn("[{}] Asset profile is null!", asset.getAssetProfileId());
} else {
ruleChainId = assetProfile.getDefaultRuleChainId();
queueName = assetProfile.getDefaultQueueName();
}
assetProfile = mainCtx.getAssetProfileCache().find(asset.getAssetProfileId());
}
return entityActionMsg(asset, asset.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, queueName, ruleChainId);
return entityActionMsg(asset, asset.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, assetProfile);
}
public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) {
RuleChainId ruleChainId = null;
String queueName = null;
HasRuleEngineProfile profile = null;
if (EntityType.DEVICE.equals(alarm.getOriginator().getEntityType())) {
DeviceId deviceId = new DeviceId(alarm.getOriginator().getId());
DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().get(getTenantId(), deviceId);
if (deviceProfile == null) {
log.warn("[{}] Device profile is null!", deviceId);
} else {
ruleChainId = deviceProfile.getDefaultRuleChainId();
queueName = deviceProfile.getDefaultQueueName();
}
profile = mainCtx.getDeviceProfileCache().get(getTenantId(), deviceId);
} else if (EntityType.ASSET.equals(alarm.getOriginator().getEntityType())) {
AssetId assetId = new AssetId(alarm.getOriginator().getId());
AssetProfile assetProfile = mainCtx.getAssetProfileCache().get(getTenantId(), assetId);
if (assetProfile == null) {
log.warn("[{}] Asset profile is null!", assetId);
} else {
ruleChainId = assetProfile.getDefaultRuleChainId();
queueName = assetProfile.getDefaultQueueName();
}
profile = mainCtx.getAssetProfileCache().get(getTenantId(), assetId);
}
return entityActionMsg(alarm, alarm.getOriginator(), ruleNodeId, action, profile);
}
public TbMsg attributesUpdatedActionMsg(EntityId originator, RuleNodeId ruleNodeId, String scope, List<AttributeKvEntry> attributes) {
ObjectNode entityNode = JacksonUtil.newObjectNode();
if (attributes != null) {
attributes.forEach(attributeKvEntry -> JacksonUtil.addKvEntry(entityNode, attributeKvEntry));
}
return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action, queueName, ruleChainId);
return attributesActionMsg(originator, ruleNodeId, scope, DataConstants.ATTRIBUTES_UPDATED, JacksonUtil.toString(entityNode));
}
public TbMsg attributesDeletedActionMsg(EntityId originator, RuleNodeId ruleNodeId, String scope, List<String> keys) {
ObjectNode entityNode = JacksonUtil.newObjectNode();
ArrayNode attrsArrayNode = entityNode.putArray("attributes");
if (keys != null) {
keys.forEach(attrsArrayNode::add);
}
return attributesActionMsg(originator, ruleNodeId, scope, DataConstants.ATTRIBUTES_DELETED, JacksonUtil.toString(entityNode));
}
private TbMsg attributesActionMsg(EntityId originator, RuleNodeId ruleNodeId, String scope, String action, String msgData) {
TbMsgMetaData tbMsgMetaData = getActionMetaData(ruleNodeId);
tbMsgMetaData.putValue("scope", scope);
HasRuleEngineProfile profile = null;
if (EntityType.DEVICE.equals(originator.getEntityType())) {
DeviceId deviceId = new DeviceId(originator.getId());
profile = mainCtx.getDeviceProfileCache().get(getTenantId(), deviceId);
} else if (EntityType.ASSET.equals(originator.getEntityType())) {
AssetId assetId = new AssetId(originator.getId());
profile = mainCtx.getAssetProfileCache().get(getTenantId(), assetId);
}
return entityActionMsg(originator, tbMsgMetaData, msgData, action, profile);
}
@Override
@ -390,17 +400,27 @@ class DefaultTbContext implements TbContext {
}
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) {
return entityActionMsg(entity, id, ruleNodeId, action, null, null);
return entityActionMsg(entity, id, ruleNodeId, action, null);
}
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action, String queueName, RuleChainId ruleChainId) {
public <E, I extends EntityId, K extends HasRuleEngineProfile> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action, K profile) {
try {
return TbMsg.newMsg(queueName, action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)), ruleChainId, null);
return entityActionMsg(id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)), action, profile);
} catch (JsonProcessingException | IllegalArgumentException e) {
throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e);
}
}
private <I extends EntityId, K extends HasRuleEngineProfile> TbMsg entityActionMsg(I id, TbMsgMetaData msgMetaData, String msgData, String action, K profile) {
String defaultQueueName = null;
RuleChainId defaultRuleChainId = null;
if (profile != null) {
defaultQueueName = profile.getDefaultQueueName();
defaultRuleChainId = profile.getDefaultRuleChainId();
}
return TbMsg.newMsg(defaultQueueName, action, id, msgMetaData, msgData, defaultRuleChainId, null);
}
@Override
public RuleNodeId getSelfId() {
return nodeCtx.getSelf().getId();

23
application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java

@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
@ -34,8 +35,6 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.DataType;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.msg.TbMsg;
@ -180,7 +179,7 @@ public class EntityActionService {
metaData.putValue(DataConstants.SCOPE, scope);
if (attributes != null) {
for (AttributeKvEntry attr : attributes) {
addKvEntry(entityNode, attr);
JacksonUtil.addKvEntry(entityNode, attr);
}
}
} else if (actionType == ActionType.ATTRIBUTES_DELETED) {
@ -258,27 +257,11 @@ public class EntityActionService {
element.put("ts", entry.getKey());
ObjectNode values = element.putObject("values");
for (TsKvEntry tsKvEntry : entry.getValue()) {
addKvEntry(values, tsKvEntry);
JacksonUtil.addKvEntry(values, tsKvEntry);
}
result.add(element);
}
}
}
private void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) throws Exception {
if (kvEntry.getDataType() == DataType.BOOLEAN) {
kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
} else if (kvEntry.getDataType() == DataType.DOUBLE) {
kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
} else if (kvEntry.getDataType() == DataType.LONG) {
kvEntry.getLongValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
} else if (kvEntry.getDataType() == DataType.JSON) {
if (kvEntry.getJsonValue().isPresent()) {
entityNode.set(kvEntry.getKey(), json.readTree(kvEntry.getJsonValue().get()));
}
} else {
entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString());
}
}
}

47
application/src/main/java/org/thingsboard/server/service/script/AbstractJsInvokeService.java

@ -15,12 +15,14 @@
*/
package org.thingsboard.server.service.script;
import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.util.Pair;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.id.CustomerId;
@ -41,13 +43,15 @@ import static java.lang.String.format;
* Created by ashvayka on 26.09.18.
*/
@Slf4j
@SuppressWarnings("UnstableApiUsage")
public abstract class AbstractJsInvokeService implements JsInvokeService {
private final TbApiUsageStateService apiUsageStateService;
private final TbApiUsageClient apiUsageClient;
protected ScheduledExecutorService timeoutExecutorService;
protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>();
protected Map<UUID, DisableListInfo> disabledFunctions = new ConcurrentHashMap<>();
protected final Map<UUID, Pair<String, String>> scriptIdToNameAndHashMap = new ConcurrentHashMap<>();
protected final Map<UUID, DisableListInfo> disabledFunctions = new ConcurrentHashMap<>();
@Getter
@Value("${js.max_total_args_size:100000}")
@ -83,27 +87,34 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
return error(format("Script body exceeds maximum allowed size of %s symbols", getMaxScriptBodySize()));
}
UUID scriptId = UUID.randomUUID();
String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_');
String scriptHash = hash(tenantId, scriptBody);
String functionName = constructFunctionName(scriptId, scriptHash);
String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames);
return doEval(scriptId, functionName, jsScript);
return doEval(scriptId, scriptHash, functionName, jsScript);
} else {
return error("JS Execution is disabled due to API limits!");
}
}
protected String constructFunctionName(UUID scriptId, String scriptHash) {
return "invokeInternal_" + scriptId.toString().replace('-', '_');
}
@Override
public ListenableFuture<String> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args) {
if (apiUsageStateService.getApiUsageState(tenantId).isJsExecEnabled()) {
String functionName = scriptIdToNameMap.get(scriptId);
if (functionName == null) {
Pair<String, String> nameAndHash = scriptIdToNameAndHashMap.get(scriptId);
if (nameAndHash == null) {
return error("No compiled script found for scriptId: [" + scriptId + "]!");
}
String functionName = nameAndHash.getFirst();
String scriptHash = nameAndHash.getSecond();
if (!isDisabled(scriptId)) {
if (argsSizeExceeded(args)) {
return scriptExecutionError(scriptId, format("Script input arguments exceed maximum allowed total args size of %s symbols", getMaxTotalArgsSize()));
}
apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.JS_EXEC_COUNT, 1);
return Futures.transformAsync(doInvokeFunction(scriptId, functionName, args), output -> {
return Futures.transformAsync(doInvokeFunction(scriptId, scriptHash, functionName, args), output -> {
String result = output.toString();
if (resultSizeExceeded(result)) {
return scriptExecutionError(scriptId, format("Script invocation result exceeds maximum allowed size of %s symbols", getMaxResultSize()));
@ -123,12 +134,12 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
@Override
public ListenableFuture<Void> release(UUID scriptId) {
String functionName = scriptIdToNameMap.get(scriptId);
if (functionName != null) {
Pair<String, String> nameAndHash = scriptIdToNameAndHashMap.get(scriptId);
if (nameAndHash != null) {
try {
scriptIdToNameMap.remove(scriptId);
scriptIdToNameAndHashMap.remove(scriptId);
disabledFunctions.remove(scriptId);
doRelease(scriptId, functionName);
doRelease(scriptId, nameAndHash.getSecond(), nameAndHash.getFirst());
} catch (Exception e) {
return Futures.immediateFailedFuture(e);
}
@ -136,16 +147,24 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
return Futures.immediateFuture(null);
}
protected abstract ListenableFuture<UUID> doEval(UUID scriptId, String functionName, String scriptBody);
protected abstract ListenableFuture<UUID> doEval(UUID scriptId, String scriptHash, String functionName, String scriptBody);
protected abstract ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args);
protected abstract ListenableFuture<Object> doInvokeFunction(UUID scriptId, String scriptHash, String functionName, Object[] args);
protected abstract void doRelease(UUID scriptId, String functionName) throws Exception;
protected abstract void doRelease(UUID scriptId, String scriptHash, String functionName) throws Exception;
protected abstract int getMaxErrors();
protected abstract long getMaxBlacklistDuration();
protected String hash(TenantId tenantId, String scriptBody) {
return Hashing.murmur3_128().newHasher()
.putLong(tenantId.getId().getMostSignificantBits())
.putLong(tenantId.getId().getLeastSignificantBits())
.putUnencodedChars(scriptBody)
.hash().toString();
}
protected void onScriptExecutionError(UUID scriptId, Throwable t, String scriptBody) {
DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo());
log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}",

10
application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java

@ -24,6 +24,7 @@ import delight.nashornsandbox.NashornSandboxes;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.util.Pair;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
@ -38,7 +39,6 @@ import javax.script.ScriptException;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
@ -121,7 +121,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
protected abstract long getMaxCpuTime();
@Override
protected ListenableFuture<UUID> doEval(UUID scriptId, String functionName, String jsScript) {
protected ListenableFuture<UUID> doEval(UUID scriptId, String scriptHash, String functionName, String jsScript) {
jsPushedMsgs.incrementAndGet();
ListenableFuture<UUID> result = jsExecutor.executeAsync(() -> {
try {
@ -135,7 +135,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
} finally {
evalLock.unlock();
}
scriptIdToNameMap.put(scriptId, functionName);
scriptIdToNameAndHashMap.put(scriptId, Pair.of(functionName, scriptHash));
return scriptId;
} catch (Exception e) {
log.debug("Failed to compile JS script: {}", e.getMessage(), e);
@ -150,7 +150,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
}
@Override
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args) {
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, String scriptHash, String functionName, Object[] args) {
jsPushedMsgs.incrementAndGet();
ListenableFuture<Object> result = jsExecutor.executeAsync(() -> {
try {
@ -174,7 +174,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
return result;
}
protected void doRelease(UUID scriptId, String functionName) throws ScriptException {
protected void doRelease(UUID scriptId, String scriptHash, String functionName) throws ScriptException {
if (useJsSandbox()) {
sandbox.eval(functionName + " = undefined;");
} else {

193
application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java

@ -18,16 +18,19 @@ package org.thingsboard.server.service.script;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.data.util.Pair;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.js.JsInvokeProtos.JsInvokeErrorCode;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -45,6 +48,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')")
@ -98,9 +103,10 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
}
@Autowired
private TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> requestTemplate;
protected TbQueueRequestTemplate<TbProtoJsQueueMsg<JsInvokeProtos.RemoteJsRequest>, TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> requestTemplate;
private Map<UUID, String> scriptIdToBodysMap = new ConcurrentHashMap<>();
protected final Map<String, String> scriptHashToBodysMap = new ConcurrentHashMap<>();
private final Lock scriptsLock = new ReentrantLock();
@PostConstruct
public void init() {
@ -117,10 +123,9 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
}
@Override
protected ListenableFuture<UUID> doEval(UUID scriptId, String functionName, String scriptBody) {
protected ListenableFuture<UUID> doEval(UUID scriptId, String scriptHash, String functionName, String scriptBody) {
JsInvokeProtos.JsCompileRequest jsRequest = JsInvokeProtos.JsCompileRequest.newBuilder()
.setScriptIdMSB(scriptId.getMostSignificantBits())
.setScriptIdLSB(scriptId.getLeastSignificantBits())
.setScriptHash(scriptHash)
.setFunctionName(functionName)
.setScriptBody(scriptBody).build();
@ -128,74 +133,100 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
.setCompileRequest(jsRequest)
.build();
log.trace("Post compile request for scriptId [{}]", scriptId);
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper));
if (maxEvalRequestsTimeout > 0) {
future = Futures.withTimeout(future, maxEvalRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
queuePushedMsgs.incrementAndGet();
Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
@Override
public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
queueEvalMsgs.incrementAndGet();
}
@Override
public void onFailure(Throwable t) {
if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
queueTimeoutMsgs.incrementAndGet();
}
queueFailedMsgs.incrementAndGet();
}
}, callbackExecutor);
log.trace("Post compile request for scriptId [{}] (hash: {})", scriptId, scriptHash);
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = sendJsRequest(UUID.randomUUID(), jsRequestWrapper, maxEvalRequestsTimeout, queueEvalMsgs);
return Futures.transform(future, response -> {
JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse();
UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB());
if (compilationResult.getSuccess()) {
scriptIdToNameMap.put(scriptId, functionName);
scriptIdToBodysMap.put(scriptId, scriptBody);
return compiledScriptId;
scriptsLock.lock();
try {
scriptIdToNameAndHashMap.put(scriptId, Pair.of(functionName, scriptHash));
scriptHashToBodysMap.put(scriptHash, scriptBody);
} finally {
scriptsLock.unlock();
}
return scriptId;
} else {
log.debug("[{}] Failed to compile script due to [{}]: {}", compiledScriptId, compilationResult.getErrorCode().name(), compilationResult.getErrorDetails());
log.debug("[{}] (hash: {}) Failed to compile script due to [{}]: {}", scriptId, compilationResult.getScriptHash(),
compilationResult.getErrorCode().name(), compilationResult.getErrorDetails());
throw new RuntimeException(compilationResult.getErrorDetails());
}
}, callbackExecutor);
}
@Override
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args) {
log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptId, maxRequestsTimeout);
final String scriptBody = scriptIdToBodysMap.get(scriptId);
if (scriptBody == null) {
return Futures.immediateFailedFuture(new RuntimeException("No script body found for scriptId: [" + scriptId + "]!"));
}
JsInvokeProtos.JsInvokeRequest.Builder jsRequestBuilder = JsInvokeProtos.JsInvokeRequest.newBuilder()
.setScriptIdMSB(scriptId.getMostSignificantBits())
.setScriptIdLSB(scriptId.getLeastSignificantBits())
.setFunctionName(functionName)
.setTimeout((int) maxExecRequestsTimeout)
.setScriptBody(scriptBody);
protected String constructFunctionName(UUID scriptId, String scriptHash) {
return "invokeInternal_" + scriptHash;
}
for (Object arg : args) {
jsRequestBuilder.addArgs(arg.toString());
@Override
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, String scriptHash, String functionName, Object[] args) {
log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptHash, maxRequestsTimeout);
String scriptBody = scriptHashToBodysMap.get(scriptHash);
if (scriptBody == null) {
return Futures.immediateFailedFuture(new RuntimeException("No script body found for script hash [" + scriptHash + "] (script id: [" + scriptId + "])"));
}
JsInvokeProtos.RemoteJsRequest jsRequestWrapper = JsInvokeProtos.RemoteJsRequest.newBuilder()
.setInvokeRequest(jsRequestBuilder.build())
.build();
JsInvokeProtos.RemoteJsRequest jsRequestWrapper = buildJsInvokeRequest(scriptHash, functionName, args, false, null);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper));
UUID requestKey = UUID.randomUUID();
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = sendJsRequest(requestKey, jsRequestWrapper, maxRequestsTimeout, queueInvokeMsgs);
return Futures.transformAsync(future, response -> {
stopWatch.stop();
log.trace("doInvokeFunction js-response took {}ms for uuid {}", stopWatch.getTotalTimeMillis(), response.getKey());
JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse();
if (invokeResult.getSuccess()) {
return Futures.immediateFuture(invokeResult.getResult());
} else {
return handleInvokeError(requestKey, scriptId, scriptHash, invokeResult.getErrorCode(),
invokeResult.getErrorDetails(), functionName, args, scriptBody);
}
}, callbackExecutor);
}
private ListenableFuture<Object> handleInvokeError(UUID requestKey, UUID scriptId, String scriptHash,
JsInvokeErrorCode errorCode, String errorDetails,
String functionName, Object[] args, String scriptBody) {
log.debug("[{}] Failed to invoke function due to [{}]: {}", scriptId, errorCode.name(), errorDetails);
RuntimeException e = new RuntimeException(errorDetails);
if (JsInvokeErrorCode.TIMEOUT_ERROR.equals(errorCode)) {
onScriptExecutionError(scriptId, e, scriptBody);
queueTimeoutMsgs.incrementAndGet();
} else if (JsInvokeErrorCode.COMPILATION_ERROR.equals(errorCode)) {
onScriptExecutionError(scriptId, e, scriptBody);
} else if (JsInvokeErrorCode.NOT_FOUND_ERROR.equals(errorCode)) {
log.debug("[{}] Remote JS executor couldn't find the script", scriptId);
if (scriptBody != null) {
JsInvokeProtos.RemoteJsRequest invokeRequestWithScriptBody = buildJsInvokeRequest(scriptHash, functionName, args, true, scriptBody);
log.debug("[{}] Sending invoke request again with script body", scriptId);
return Futures.transformAsync(sendJsRequest(requestKey, invokeRequestWithScriptBody, maxRequestsTimeout, queueInvokeMsgs), r -> {
JsInvokeProtos.JsInvokeResponse result = r.getValue().getInvokeResponse();
if (result.getSuccess()) {
return Futures.immediateFuture(result.getResult());
} else {
return handleInvokeError(requestKey, scriptId, scriptHash, result.getErrorCode(),
result.getErrorDetails(), functionName, args, null);
}
}, MoreExecutors.directExecutor());
}
}
queueFailedMsgs.incrementAndGet();
return Futures.immediateFailedFuture(e);
}
private ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> sendJsRequest(UUID requestKey, JsInvokeProtos.RemoteJsRequest jsRequestWrapper,
long maxRequestsTimeout, AtomicInteger msgsCounter) {
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = requestTemplate.send(new TbProtoJsQueueMsg<>(requestKey, jsRequestWrapper));
if (maxRequestsTimeout > 0) {
future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
queuePushedMsgs.incrementAndGet();
Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
Futures.addCallback(future, new FutureCallback<>() {
@Override
public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
queueInvokeMsgs.incrementAndGet();
msgsCounter.incrementAndGet();
}
@Override
@ -206,32 +237,32 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
queueFailedMsgs.incrementAndGet();
}
}, callbackExecutor);
return Futures.transform(future, response -> {
stopWatch.stop();
log.trace("doInvokeFunction js-response took {}ms for uuid {}", stopWatch.getTotalTimeMillis(), response.getKey());
JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse();
if (invokeResult.getSuccess()) {
return invokeResult.getResult();
} else {
final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails());
if (JsInvokeProtos.JsInvokeErrorCode.TIMEOUT_ERROR.equals(invokeResult.getErrorCode())) {
onScriptExecutionError(scriptId, e, scriptBody);
queueTimeoutMsgs.incrementAndGet();
} else if (JsInvokeProtos.JsInvokeErrorCode.COMPILATION_ERROR.equals(invokeResult.getErrorCode())) {
onScriptExecutionError(scriptId, e, scriptBody);
}
queueFailedMsgs.incrementAndGet();
log.debug("[{}] Failed to invoke function due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails());
throw e;
}
}, callbackExecutor);
return future;
}
private JsInvokeProtos.RemoteJsRequest buildJsInvokeRequest(String scriptHash, String functionName, Object[] args, boolean includeScriptBody, String scriptBody) {
JsInvokeProtos.JsInvokeRequest.Builder jsRequestBuilder = JsInvokeProtos.JsInvokeRequest.newBuilder()
.setScriptHash(scriptHash)
.setFunctionName(functionName)
.setTimeout((int) maxExecRequestsTimeout);
if (includeScriptBody) jsRequestBuilder.setScriptBody(scriptBody);
for (Object arg : args) {
jsRequestBuilder.addArgs(arg.toString());
}
JsInvokeProtos.RemoteJsRequest jsRequestWrapper = JsInvokeProtos.RemoteJsRequest.newBuilder()
.setInvokeRequest(jsRequestBuilder.build())
.build();
return jsRequestWrapper;
}
@Override
protected void doRelease(UUID scriptId, String functionName) throws Exception {
protected void doRelease(UUID scriptId, String scriptHash, String functionName) throws Exception {
if (scriptIdToNameAndHashMap.values().stream().map(Pair::getSecond).anyMatch(hash -> hash.equals(scriptHash))) {
return;
}
JsInvokeProtos.JsReleaseRequest jsRequest = JsInvokeProtos.JsReleaseRequest.newBuilder()
.setScriptIdMSB(scriptId.getMostSignificantBits())
.setScriptIdLSB(scriptId.getLeastSignificantBits())
.setScriptHash(scriptHash)
.setFunctionName(functionName).build();
JsInvokeProtos.RemoteJsRequest jsRequestWrapper = JsInvokeProtos.RemoteJsRequest.newBuilder()
@ -244,12 +275,18 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
}
JsInvokeProtos.RemoteJsResponse response = future.get().getValue();
JsInvokeProtos.JsReleaseResponse compilationResult = response.getReleaseResponse();
UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB());
if (compilationResult.getSuccess()) {
scriptIdToBodysMap.remove(scriptId);
JsInvokeProtos.JsReleaseResponse releaseResponse = response.getReleaseResponse();
if (releaseResponse.getSuccess()) {
scriptsLock.lock();
try {
if (scriptIdToNameAndHashMap.values().stream().map(Pair::getSecond).noneMatch(h -> h.equals(scriptHash))) {
scriptHashToBodysMap.remove(scriptHash);
}
} finally {
scriptsLock.unlock();
}
} else {
log.debug("[{}] Failed to release script due", compiledScriptId);
log.debug("[{}] Failed to release script", scriptHash);
}
}

2
application/src/main/resources/thingsboard.yml

@ -270,7 +270,7 @@ sql:
notifications:
partition_size: "${SQL_NOTIFICATIONS_PARTITION_SIZE_HOURS:168}"
# Specify whether to sort entities before batch update. Should be enabled for cluster mode to avoid deadlocks
batch_sort: "${SQL_BATCH_SORT:false}"
batch_sort: "${SQL_BATCH_SORT:true}"
# Specify whether to remove null characters from strValue of attributes and timeseries before insert
remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
# Specify whether to log database queries and their parameters generated by entity query repository

2
application/src/test/java/org/thingsboard/server/service/script/JsInvokeServiceTest.java → application/src/test/java/org/thingsboard/server/service/script/LocalJsInvokeServiceTest.java

@ -35,7 +35,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
"js.max_result_size=50",
"js.local.max_errors=2"
})
class JsInvokeServiceTest extends AbstractControllerTest {
class LocalJsInvokeServiceTest extends AbstractControllerTest {
@Autowired
private NashornJsInvokeService jsInvokeService;

219
application/src/test/java/org/thingsboard/server/service/script/RemoteJsInvokeServiceTest.java

@ -0,0 +1,219 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.script;
import com.google.common.util.concurrent.Futures;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest;
import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class RemoteJsInvokeServiceTest {
private RemoteJsInvokeService remoteJsInvokeService;
private TbQueueRequestTemplate<TbProtoJsQueueMsg<RemoteJsRequest>, TbProtoQueueMsg<RemoteJsResponse>> jsRequestTemplate;
@BeforeEach
public void beforeEach() {
TbApiUsageStateService apiUsageStateService = mock(TbApiUsageStateService.class);
ApiUsageState apiUsageState = mock(ApiUsageState.class);
when(apiUsageState.isJsExecEnabled()).thenReturn(true);
when(apiUsageStateService.getApiUsageState(any())).thenReturn(apiUsageState);
TbApiUsageClient apiUsageClient = mock(TbApiUsageClient.class);
remoteJsInvokeService = new RemoteJsInvokeService(apiUsageStateService, apiUsageClient);
jsRequestTemplate = mock(TbQueueRequestTemplate.class);
remoteJsInvokeService.requestTemplate = jsRequestTemplate;
}
@AfterEach
public void afterEach() {
reset(jsRequestTemplate);
}
@Test
public void whenInvokingFunction_thenDoNotSendScriptBody() throws Exception {
mockJsEvalResponse();
String scriptBody = "return { a: 'b'};";
UUID scriptId = remoteJsInvokeService.eval(TenantId.SYS_TENANT_ID, JsScriptType.RULE_NODE_SCRIPT, scriptBody).get();
reset(jsRequestTemplate);
String expectedInvocationResult = "scriptInvocationResult";
doReturn(Futures.immediateFuture(new TbProtoJsQueueMsg<>(UUID.randomUUID(), RemoteJsResponse.newBuilder()
.setInvokeResponse(JsInvokeProtos.JsInvokeResponse.newBuilder()
.setSuccess(true)
.setResult(expectedInvocationResult)
.build())
.build())))
.when(jsRequestTemplate).send(any());
ArgumentCaptor<TbProtoJsQueueMsg<RemoteJsRequest>> jsRequestCaptor = ArgumentCaptor.forClass(TbProtoJsQueueMsg.class);
Object invocationResult = remoteJsInvokeService.invokeFunction(TenantId.SYS_TENANT_ID, null, scriptId, "{}").get();
verify(jsRequestTemplate).send(jsRequestCaptor.capture());
JsInvokeProtos.JsInvokeRequest jsInvokeRequestMade = jsRequestCaptor.getValue().getValue().getInvokeRequest();
assertThat(jsInvokeRequestMade.getScriptBody()).isNullOrEmpty();
assertThat(jsInvokeRequestMade.getScriptHash()).isEqualTo(getScriptHash(scriptId));
assertThat(invocationResult).isEqualTo(expectedInvocationResult);
}
@Test
public void whenInvokingFunctionAndRemoteJsExecutorRemovedScript_thenHandleNotFoundErrorAndMakeInvokeRequestWithScriptBody() throws Exception {
mockJsEvalResponse();
String scriptBody = "return { a: 'b'};";
UUID scriptId = remoteJsInvokeService.eval(TenantId.SYS_TENANT_ID, JsScriptType.RULE_NODE_SCRIPT, scriptBody).get();
reset(jsRequestTemplate);
doReturn(Futures.immediateFuture(new TbProtoJsQueueMsg<>(UUID.randomUUID(), RemoteJsResponse.newBuilder()
.setInvokeResponse(JsInvokeProtos.JsInvokeResponse.newBuilder()
.setSuccess(false)
.setErrorCode(JsInvokeProtos.JsInvokeErrorCode.NOT_FOUND_ERROR)
.build())
.build())))
.when(jsRequestTemplate).send(argThat(jsQueueMsg -> {
return StringUtils.isEmpty(jsQueueMsg.getValue().getInvokeRequest().getScriptBody());
}));
String expectedInvocationResult = "invocationResult";
doReturn(Futures.immediateFuture(new TbProtoJsQueueMsg<>(UUID.randomUUID(), RemoteJsResponse.newBuilder()
.setInvokeResponse(JsInvokeProtos.JsInvokeResponse.newBuilder()
.setSuccess(true)
.setResult(expectedInvocationResult)
.build())
.build())))
.when(jsRequestTemplate).send(argThat(jsQueueMsg -> {
return StringUtils.isNotEmpty(jsQueueMsg.getValue().getInvokeRequest().getScriptBody());
}));
ArgumentCaptor<TbProtoJsQueueMsg<RemoteJsRequest>> jsRequestsCaptor = ArgumentCaptor.forClass(TbProtoJsQueueMsg.class);
Object invocationResult = remoteJsInvokeService.invokeFunction(TenantId.SYS_TENANT_ID, null, scriptId, "{}").get();
verify(jsRequestTemplate, times(2)).send(jsRequestsCaptor.capture());
List<TbProtoJsQueueMsg<RemoteJsRequest>> jsInvokeRequestsMade = jsRequestsCaptor.getAllValues();
JsInvokeProtos.JsInvokeRequest firstRequestMade = jsInvokeRequestsMade.get(0).getValue().getInvokeRequest();
assertThat(firstRequestMade.getScriptBody()).isNullOrEmpty();
JsInvokeProtos.JsInvokeRequest secondRequestMade = jsInvokeRequestsMade.get(1).getValue().getInvokeRequest();
assertThat(secondRequestMade.getScriptBody()).contains(scriptBody);
assertThat(jsInvokeRequestsMade.stream().map(TbProtoQueueMsg::getKey).distinct().count()).as("partition keys are same")
.isOne();
assertThat(invocationResult).isEqualTo(expectedInvocationResult);
}
@Test
public void whenDoingEval_thenSaveScriptByHashOfTenantIdAndScriptBody() throws Exception {
mockJsEvalResponse();
TenantId tenantId1 = TenantId.fromUUID(UUID.randomUUID());
String scriptBody1 = "var msg = { temp: 42, humidity: 77 };\n" +
"var metadata = { data: 40 };\n" +
"var msgType = \"POST_TELEMETRY_REQUEST\";\n" +
"\n" +
"return { msg: msg, metadata: metadata, msgType: msgType };";
Set<String> scriptHashes = new HashSet<>();
String tenant1Script1Hash = null;
for (int i = 0; i < 3; i++) {
UUID scriptUuid = remoteJsInvokeService.eval(tenantId1, JsScriptType.RULE_NODE_SCRIPT, scriptBody1).get();
tenant1Script1Hash = getScriptHash(scriptUuid);
scriptHashes.add(tenant1Script1Hash);
}
assertThat(scriptHashes).as("Unique scripts ids").size().isOne();
TenantId tenantId2 = TenantId.fromUUID(UUID.randomUUID());
UUID scriptUuid = remoteJsInvokeService.eval(tenantId2, JsScriptType.RULE_NODE_SCRIPT, scriptBody1).get();
String tenant2Script1Id = getScriptHash(scriptUuid);
assertThat(tenant2Script1Id).isNotEqualTo(tenant1Script1Hash);
String scriptBody2 = scriptBody1 + ";;";
scriptUuid = remoteJsInvokeService.eval(tenantId2, JsScriptType.RULE_NODE_SCRIPT, scriptBody2).get();
String tenant2Script2Id = getScriptHash(scriptUuid);
assertThat(tenant2Script2Id).isNotEqualTo(tenant2Script1Id);
}
@Test
public void whenReleasingScript_thenCheckForHashUsages() throws Exception {
mockJsEvalResponse();
String scriptBody = "return { a: 'b'};";
UUID scriptId1 = remoteJsInvokeService.eval(TenantId.SYS_TENANT_ID, JsScriptType.RULE_NODE_SCRIPT, scriptBody).get();
UUID scriptId2 = remoteJsInvokeService.eval(TenantId.SYS_TENANT_ID, JsScriptType.RULE_NODE_SCRIPT, scriptBody).get();
String scriptHash = getScriptHash(scriptId1);
assertThat(scriptHash).isEqualTo(getScriptHash(scriptId2));
reset(jsRequestTemplate);
doReturn(Futures.immediateFuture(new TbProtoQueueMsg<>(UUID.randomUUID(), RemoteJsResponse.newBuilder()
.setReleaseResponse(JsInvokeProtos.JsReleaseResponse.newBuilder()
.setSuccess(true)
.build())
.build())))
.when(jsRequestTemplate).send(any());
remoteJsInvokeService.release(scriptId1).get();
verifyNoInteractions(jsRequestTemplate);
assertThat(remoteJsInvokeService.scriptHashToBodysMap).containsKey(scriptHash);
remoteJsInvokeService.release(scriptId2).get();
verify(jsRequestTemplate).send(any());
assertThat(remoteJsInvokeService.scriptHashToBodysMap).isEmpty();
}
private String getScriptHash(UUID scriptUuid) {
return remoteJsInvokeService.scriptIdToNameAndHashMap.get(scriptUuid).getSecond();
}
private void mockJsEvalResponse() {
doAnswer(methodCall -> Futures.immediateFuture(new TbProtoJsQueueMsg<>(UUID.randomUUID(), RemoteJsResponse.newBuilder()
.setCompileResponse(JsInvokeProtos.JsCompileResponse.newBuilder()
.setSuccess(true)
.setScriptHash(methodCall.<TbProtoQueueMsg<RemoteJsRequest>>getArgument(0).getValue().getCompileRequest().getScriptHash())
.build())
.build())))
.when(jsRequestTemplate).send(argThat(jsQueueMsg -> jsQueueMsg.getValue().hasCompileRequest()));
}
}

17
common/cluster-api/src/main/proto/jsinvoke.proto

@ -23,6 +23,7 @@ enum JsInvokeErrorCode {
COMPILATION_ERROR = 0;
RUNTIME_ERROR = 1;
TIMEOUT_ERROR = 2;
NOT_FOUND_ERROR = 3;
}
message RemoteJsRequest {
@ -40,39 +41,34 @@ message RemoteJsResponse {
}
message JsCompileRequest {
int64 scriptIdMSB = 1;
int64 scriptIdLSB = 2;
string functionName = 3;
string scriptBody = 4;
string scriptHash = 5;
}
message JsReleaseRequest {
int64 scriptIdMSB = 1;
int64 scriptIdLSB = 2;
string functionName = 3;
string scriptHash = 4;
}
message JsReleaseResponse {
bool success = 1;
int64 scriptIdMSB = 2;
int64 scriptIdLSB = 3;
string scriptHash = 4;
}
message JsCompileResponse {
bool success = 1;
int64 scriptIdMSB = 2;
int64 scriptIdLSB = 3;
JsInvokeErrorCode errorCode = 4;
string errorDetails = 5;
string scriptHash = 6;
}
message JsInvokeRequest {
int64 scriptIdMSB = 1;
int64 scriptIdLSB = 2;
string functionName = 3;
string scriptBody = 4;
int32 timeout = 5;
repeated string args = 6;
string scriptHash = 7;
}
message JsInvokeResponse {
@ -81,4 +77,3 @@ message JsInvokeResponse {
JsInvokeErrorCode errorCode = 3;
string errorDetails = 4;
}

4
common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java

@ -45,7 +45,7 @@ import static org.thingsboard.server.common.data.SearchTextBasedWithAdditionalIn
@ToString(exclude = {"image", "profileDataBytes"})
@EqualsAndHashCode(callSuper = true)
@Slf4j
public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements HasName, HasTenantId, HasOtaPackage, ExportableEntity<DeviceProfileId> {
public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements HasName, HasTenantId, HasOtaPackage, HasRuleEngineProfile, ExportableEntity<DeviceProfileId> {
private static final long serialVersionUID = 6998485460273302018L;
@ -141,7 +141,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
}
@ApiModelProperty(position = 5, value = "Used to mark the default profile. Default profile is used when the device profile is not specified during device creation.")
public boolean isDefault(){
public boolean isDefault() {
return isDefault;
}

26
common/data/src/main/java/org/thingsboard/server/common/data/HasRuleEngineProfile.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data;
import org.thingsboard.server.common.data.id.RuleChainId;
public interface HasRuleEngineProfile {
RuleChainId getDefaultRuleChainId();
String getDefaultQueueName();
}

10
common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java

@ -146,6 +146,16 @@ public class StringUtils {
return org.apache.commons.lang3.StringUtils.substringAfterLast(str, sep);
}
public static boolean containedByAny(String searchString, String... strings) {
if (searchString == null) return false;
for (String string : strings) {
if (string != null && string.contains(searchString)) {
return true;
}
}
return false;
}
public static String randomNumeric(int length) {
return RandomStringUtils.randomNumeric(length);
}

3
common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java

@ -23,6 +23,7 @@ import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasRuleEngineProfile;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.SearchTextBased;
import org.thingsboard.server.common.data.id.AssetProfileId;
@ -37,7 +38,7 @@ import org.thingsboard.server.common.data.validation.NoXss;
@ToString(exclude = {"image"})
@EqualsAndHashCode(callSuper = true)
@Slf4j
public class AssetProfile extends SearchTextBased<AssetProfileId> implements HasName, HasTenantId, ExportableEntity<AssetProfileId> {
public class AssetProfile extends SearchTextBased<AssetProfileId> implements HasName, HasTenantId, HasRuleEngineProfile, ExportableEntity<AssetProfileId> {
private static final long serialVersionUID = 6998485460273302018L;

18
common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java

@ -24,6 +24,8 @@ import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.thingsboard.server.common.data.kv.DataType;
import org.thingsboard.server.common.data.kv.KvEntry;
import java.io.IOException;
import java.util.ArrayList;
@ -212,4 +214,20 @@ public class JacksonUtil {
}
}
}
public static void addKvEntry(ObjectNode entityNode, KvEntry kvEntry) {
if (kvEntry.getDataType() == DataType.BOOLEAN) {
kvEntry.getBooleanValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
} else if (kvEntry.getDataType() == DataType.DOUBLE) {
kvEntry.getDoubleValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
} else if (kvEntry.getDataType() == DataType.LONG) {
kvEntry.getLongValue().ifPresent(value -> entityNode.put(kvEntry.getKey(), value));
} else if (kvEntry.getDataType() == DataType.JSON) {
if (kvEntry.getJsonValue().isPresent()) {
entityNode.set(kvEntry.getKey(), JacksonUtil.toJsonNode(kvEntry.getJsonValue().get()));
}
} else {
entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString());
}
}
}

45
dao/src/main/java/org/thingsboard/server/dao/aspect/SqlDaoCallsAspect.java

@ -21,7 +21,6 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
@ -29,9 +28,11 @@ import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.reflect.MethodSignature;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.hibernate.exception.JDBCConnectionException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.Arrays;
@ -46,6 +47,8 @@ import java.util.concurrent.ConcurrentMap;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.join;
@Aspect
@ConditionalOnProperty(prefix = "sql", value = "log_tenant_stats", havingValue = "true")
@Component
@ -55,6 +58,12 @@ public class SqlDaoCallsAspect {
private final Set<String> invalidTenantDbCallMethods = ConcurrentHashMap.newKeySet();
private final ConcurrentMap<TenantId, DbCallStats> statsMap = new ConcurrentHashMap<>();
@Value("${sql.batch_sort:true}")
private boolean batchSortEnabled;
private static final String DEADLOCK_DETECTED_ERROR = "deadlock detected";
@Scheduled(initialDelayString = "${sql.log_tenant_stats_interval_ms:60000}",
fixedDelayString = "${sql.log_tenant_stats_interval_ms:60000}")
public void printStats() {
@ -135,7 +144,7 @@ public class SqlDaoCallsAspect {
@SuppressWarnings({"rawtypes", "unchecked"})
@Around("@within(org.thingsboard.server.dao.util.SqlDao)")
public Object logExecutionTime(ProceedingJoinPoint joinPoint) throws Throwable {
public Object handleSqlCall(ProceedingJoinPoint joinPoint) throws Throwable {
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
var methodName = signature.toShortString();
if (invalidTenantDbCallMethods.contains(methodName)) {
@ -155,29 +164,47 @@ public class SqlDaoCallsAspect {
new FutureCallback<>() {
@Override
public void onSuccess(@Nullable Object result) {
logTenantMethodExecution(tenantId, methodName, true, startTime, null);
reportSuccessfulMethodExecution(tenantId, methodName, startTime);
}
@Override
public void onFailure(Throwable t) {
logTenantMethodExecution(tenantId, methodName, false, startTime, t);
reportFailedMethodExecution(tenantId, methodName, startTime, t, joinPoint);
}
},
MoreExecutors.directExecutor());
} else {
logTenantMethodExecution(tenantId, methodName, true, startTime, null);
reportSuccessfulMethodExecution(tenantId, methodName, startTime);
}
return result;
} catch (Throwable t) {
logTenantMethodExecution(tenantId, methodName, false, startTime, t);
reportFailedMethodExecution(tenantId, methodName, startTime, t, joinPoint);
throw t;
}
}
private void logTenantMethodExecution(TenantId tenantId, String method, boolean success, long startTime, Throwable t) {
if (!success && ExceptionUtils.indexOfThrowable(t, JDBCConnectionException.class) >= 0) {
return;
private void reportFailedMethodExecution(TenantId tenantId, String method, long startTime, Throwable t, ProceedingJoinPoint joinPoint) {
if (t != null) {
if (ExceptionUtils.indexOfThrowable(t, JDBCConnectionException.class) >= 0) {
return;
}
if (StringUtils.containedByAny(DEADLOCK_DETECTED_ERROR, ExceptionUtils.getRootCauseMessage(t), ExceptionUtils.getMessage(t))) {
if (!batchSortEnabled) {
log.warn("Deadlock was detected for method {} (tenant: {}). You might need to enable 'sql.batch_sort' option.", method, tenantId);
} else {
log.error("Deadlock was detected for method {} (tenant: {}). Arguments passed: \n{}\n The error: ",
method, tenantId, join(joinPoint.getArgs(), System.lineSeparator()), t);
}
}
}
reportMethodExecution(tenantId, method, false, startTime);
}
private void reportSuccessfulMethodExecution(TenantId tenantId, String method, long startTime) {
reportMethodExecution(tenantId, method, true, startTime);
}
private void reportMethodExecution(TenantId tenantId, String method, boolean success, long startTime) {
statsMap.computeIfAbsent(tenantId, DbCallStats::new)
.onMethodCall(method, success, System.currentTimeMillis() - startTime);
}

4
dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java

@ -25,6 +25,7 @@ import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
import org.thingsboard.server.dao.util.SqlDao;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@ -35,6 +36,7 @@ import java.util.regex.Pattern;
@Repository
@Slf4j
@SqlDao
public abstract class AttributeKvInsertRepository {
private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE)));
@ -58,7 +60,7 @@ public abstract class AttributeKvInsertRepository {
@Value("${sql.remove_null_chars:true}")
private boolean removeNullChars;
protected void saveOrUpdate(List<AttributeKvEntity> entities) {
public void saveOrUpdate(List<AttributeKvEntity> entities) {
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus status) {

2
dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java

@ -78,7 +78,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
@Value("${sql.attributes.batch_threads:4}")
private int batchThreads;
@Value("${sql.batch_sort:false}")
@Value("${sql.batch_sort:true}")
private boolean batchSortEnabled;
private TbSqlBlockingQueueWrapper<AttributeKvEntity> queue;

2
dao/src/main/java/org/thingsboard/server/dao/sql/attributes/SqlAttributesInsertRepository.java

@ -17,9 +17,11 @@ package org.thingsboard.server.dao.sql.attributes;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.dao.util.SqlDao;
@Repository
@Transactional
@SqlDao
public class SqlAttributesInsertRepository extends AttributeKvInsertRepository {
}

4
dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java

@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.common.data.event.RuleChainDebugEvent;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.event.StatisticsEvent;
import org.thingsboard.server.dao.util.SqlDao;
import javax.annotation.PostConstruct;
import java.sql.PreparedStatement;
@ -45,6 +46,7 @@ import java.util.stream.Collectors;
@Repository
@Transactional
@SqlDao
public class EventInsertRepository {
private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE)));
@ -81,7 +83,7 @@ public class EventInsertRepository {
"VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;");
}
protected void save(List<Event> entities) {
public void save(List<Event> entities) {
Map<EventType, List<Event>> eventsByType = entities.stream().collect(Collectors.groupingBy(Event::getType, Collectors.toList()));
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override

2
dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java

@ -106,7 +106,7 @@ public class JpaBaseEventDao implements EventDao {
@Value("${sql.events.batch_threads:3}")
private int batchThreads;
@Value("${sql.batch_sort:false}")
@Value("${sql.batch_sort:true}")
private boolean batchSortEnabled;
private TbSqlBlockingQueueWrapper<Event> queue;

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java

@ -63,7 +63,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries
@Value("${sql.timescale.batch_threads:4}")
protected int timescaleBatchThreads;
@Value("${sql.batch_sort:false}")
@Value("${sql.batch_sort:true}")
protected boolean batchSortEnabled;
@Value("${sql.ttl.ts.ts_key_value_ttl:0}")

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java

@ -94,7 +94,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme
@Value("${sql.ts_latest.batch_threads:4}")
private int tsLatestBatchThreads;
@Value("${sql.batch_sort:false}")
@Value("${sql.batch_sort:true}")
protected boolean batchSortEnabled;
@Autowired

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/sql/SqlLatestInsertTsRepository.java

@ -24,6 +24,7 @@ import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository;
import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository;
import org.thingsboard.server.dao.util.SqlDao;
import org.thingsboard.server.dao.util.SqlTsLatestAnyDao;
import java.sql.PreparedStatement;
@ -36,6 +37,7 @@ import java.util.List;
@SqlTsLatestAnyDao
@Repository
@Transactional
@SqlDao
public class SqlLatestInsertTsRepository extends AbstractInsertRepository implements InsertLatestTsRepository {
@Value("${sql.ts_latest.update_by_latest_ts:true}")

5
msa/js-executor/api/jsExecutor.models.ts

@ -16,8 +16,9 @@
export interface TbMessage {
scriptIdMSB: string;
scriptIdLSB: string;
scriptIdMSB: string; // deprecated
scriptIdLSB: string; // deprecated
scriptHash: string;
}
export interface RemoteJsRequest {

82
msa/js-executor/api/jsInvokeMessageProcessor.ts

@ -18,7 +18,7 @@ import config from 'config';
import { _logger } from '../config/logger';
import { JsExecutor, TbScript } from './jsExecutor';
import { performance } from 'perf_hooks';
import { isString, parseJsErrorDetails, toUUIDString, UUIDFromBuffer, UUIDToBits } from './utils';
import { isString, parseJsErrorDetails, toUUIDString, UUIDFromBuffer, UUIDToBits, isNotUUID } from './utils';
import { IQueue } from '../queue/queue.models';
import {
JsCompileRequest,
@ -36,6 +36,7 @@ import Long from 'long';
const COMPILATION_ERROR = 0;
const RUNTIME_ERROR = 1;
const TIMEOUT_ERROR = 2;
const NOT_FOUND_ERROR = 3;
const statFrequency = Number(config.get('script.stat_print_frequency'));
const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency'));
@ -129,7 +130,12 @@ export class JsInvokeMessageProcessor {
processCompileRequest(requestId: string, responseTopic: string, headers: any, compileRequest: JsCompileRequest) {
const scriptId = JsInvokeMessageProcessor.getScriptId(compileRequest);
this.logger.debug('[%s] Processing compile request, scriptId: [%s]', requestId, scriptId);
if (this.scriptMap.has(scriptId)) {
const compileResponse = JsInvokeMessageProcessor.createCompileResponse(scriptId, true);
this.logger.debug('[%s] Script was already compiled, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, compileResponse);
return;
}
this.executor.compileScript(compileRequest.scriptBody).then(
(script) => {
this.cacheScript(scriptId, script);
@ -170,7 +176,7 @@ export class JsInvokeMessageProcessor {
this.logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
} else {
let err = {
const err = {
name: 'Error',
message: 'script invocation result exceeds maximum allowed size of ' + maxResultSize + ' symbols'
}
@ -193,8 +199,12 @@ export class JsInvokeMessageProcessor {
)
},
(err: any) => {
const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse("", false, COMPILATION_ERROR, err);
this.logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR);
let errorCode = COMPILATION_ERROR;
if (err?.name === 'script body not found') {
errorCode = NOT_FOUND_ERROR;
}
const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse("", false, errorCode, err);
this.logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode);
this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse);
}
);
@ -222,7 +232,7 @@ export class JsInvokeMessageProcessor {
const remoteResponse = JsInvokeMessageProcessor.createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse);
const rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8');
this.logger.debug('[%s] Sending response to queue, scriptId: [%s]', requestId, scriptId);
this.producer.send(responseTopic, scriptId, rawResponse, headers).then(
this.producer.send(responseTopic, requestId, rawResponse, headers).then(
() => {
this.logger.debug('[%s] Response sent to queue, took [%s]ms, scriptId: [%s]', requestId, (performance.now() - tStartSending), scriptId);
},
@ -242,7 +252,7 @@ export class JsInvokeMessageProcessor {
if (script) {
self.incrementUseScriptId(scriptId);
resolve(script);
} else {
} else if (scriptBody) {
const startTime = performance.now();
self.executor.compileScript(scriptBody).then(
(compiledScript) => {
@ -255,6 +265,12 @@ export class JsInvokeMessageProcessor {
reject(err);
}
);
} else {
const err = {
name: 'script body not found',
message: ''
}
reject(err);
}
});
}
@ -285,14 +301,26 @@ export class JsInvokeMessageProcessor {
}
private static createCompileResponse(scriptId: string, success: boolean, errorCode?: number, err?: any): JsCompileResponse {
const scriptIdBits = UUIDToBits(scriptId);
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
};
if (isNotUUID(scriptId)) {
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
scriptIdMSB: "0",
scriptIdLSB: "0",
scriptHash: scriptId
};
} else { // this is for backward compatibility (to be able to work with tb-node of previous version) - todo: remove in the next release
let scriptIdBits = UUIDToBits(scriptId);
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1],
scriptHash: ""
};
}
}
private static createInvokeResponse(result: string, success: boolean, errorCode?: number, err?: any): JsInvokeResponse {
@ -305,16 +333,26 @@ export class JsInvokeMessageProcessor {
}
private static createReleaseResponse(scriptId: string, success: boolean): JsReleaseResponse {
const scriptIdBits = UUIDToBits(scriptId);
return {
success: success,
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
};
if (isNotUUID(scriptId)) {
return {
success: success,
scriptIdMSB: "0",
scriptIdLSB: "0",
scriptHash: scriptId,
};
} else { // todo: remove in the next release
let scriptIdBits = UUIDToBits(scriptId);
return {
success: success,
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1],
scriptHash: ""
}
}
}
private static getScriptId(request: TbMessage): string {
return toUUIDString(request.scriptIdMSB, request.scriptIdLSB);
return request.scriptHash ? request.scriptHash : toUUIDString(request.scriptIdMSB, request.scriptIdLSB);
}
private incrementUseScriptId(scriptId: string) {

4
msa/js-executor/api/utils.ts

@ -58,3 +58,7 @@ export function parseJsErrorDetails(err: any): string | undefined {
}
return details;
}
export function isNotUUID(candidate: string) {
return candidate.length != 36 || !candidate.includes('-');
}

4
msa/js-executor/queue/awsSqsTemplate.ts

@ -123,10 +123,10 @@ export class AwsSqsTemplate implements IQueue {
this.timer = setTimeout(() => {this.getAndProcessMessage(messageProcessor, params)}, this.pollInterval);
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
let msgBody = JSON.stringify(
{
key: scriptId,
key: msgKey,
data: [...rawResponse],
headers: headers
});

5
msa/js-executor/queue/kafkaTemplate.ts

@ -149,12 +149,11 @@ export class KafkaTemplate implements IQueue {
});
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
this.logger.debug('Pending queue response, scriptId: [%s]', scriptId);
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
const message = {
topic: responseTopic,
messages: [{
key: scriptId,
key: msgKey,
value: rawResponse,
headers: headers.data
}]

4
msa/js-executor/queue/pubSubTemplate.ts

@ -80,7 +80,7 @@ export class PubSubTemplate implements IQueue {
subscription.on('message', messageHandler);
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
if (!(this.subscriptions.includes(responseTopic) && this.topics.includes(this.requestTopic))) {
await this.createTopic(this.requestTopic);
await this.createSubscription(this.requestTopic);
@ -88,7 +88,7 @@ export class PubSubTemplate implements IQueue {
let data = JSON.stringify(
{
key: scriptId,
key: msgKey,
data: [...rawResponse],
headers: headers
});

2
msa/js-executor/queue/queue.models.ts

@ -17,6 +17,6 @@
export interface IQueue {
name: string;
init(): Promise<void>;
send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any>;
send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any>;
destroy(): Promise<void>;
}

4
msa/js-executor/queue/rabbitmqTemplate.ts

@ -65,7 +65,7 @@ export class RabbitMqTemplate implements IQueue {
})
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
if (!this.topics.includes(responseTopic)) {
await this.createQueue(responseTopic);
@ -74,7 +74,7 @@ export class RabbitMqTemplate implements IQueue {
let data = JSON.stringify(
{
key: scriptId,
key: msgKey,
data: [...rawResponse],
headers: headers
});

4
msa/js-executor/queue/serviceBusTemplate.ts

@ -82,7 +82,7 @@ export class ServiceBusTemplate implements IQueue {
this.receiver.subscribe({processMessage: messageHandler, processError: errorHandler})
}
async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise<any> {
if (!this.queues.includes(this.requestTopic)) {
await this.createQueueIfNotExist(this.requestTopic);
this.queues.push(this.requestTopic);
@ -96,7 +96,7 @@ export class ServiceBusTemplate implements IQueue {
}
let data = {
key: scriptId,
key: msgKey,
data: [...rawResponse],
headers: headers
};

7
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java

@ -31,10 +31,10 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleNode;
@ -62,6 +62,7 @@ import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.user.UserService;
import java.util.List;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
@ -183,6 +184,10 @@ public interface TbContext {
// TODO: Does this changes the message?
TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action);
TbMsg attributesUpdatedActionMsg(EntityId originator, RuleNodeId ruleNodeId, String scope, List<AttributeKvEntry> attributes);
TbMsg attributesDeletedActionMsg(EntityId originator, RuleNodeId ruleNodeId, String scope, List<String> keys);
void onEdgeEventUpdate(TenantId tenantId, EdgeId edgeId);
/*

47
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java

@ -21,6 +21,7 @@ import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.thingsboard.common.util.CollectionsUtil;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.rule.engine.api.EmptyNodeConfiguration;
@ -74,6 +75,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode {
if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) ||
DataConstants.ATTRIBUTES_DELETED.equals(msg.getType()) ||
DataConstants.ACTIVITY_EVENT.equals(msg.getType()) ||
DataConstants.INACTIVITY_EVENT.equals(msg.getType()) ||
SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) {
if (!msg.getMetaData().getData().isEmpty()) {
long now = System.currentTimeMillis();
@ -89,25 +91,7 @@ public class TbCopyAttributesToEntityViewNode implements TbNode {
long startTime = entityView.getStartTimeMs();
long endTime = entityView.getEndTimeMs();
if ((endTime != 0 && endTime > now && startTime < now) || (endTime == 0 && startTime < now)) {
if (DataConstants.ATTRIBUTES_UPDATED.equals(msg.getType()) ||
DataConstants.ACTIVITY_EVENT.equals(msg.getType()) ||
SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) {
Set<AttributeKvEntry> attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData()));
List<AttributeKvEntry> filteredAttributes =
attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList());
ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes,
new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void result) {
transformAndTellNext(ctx, msg, entityView);
}
@Override
public void onFailure(Throwable t) {
ctx.tellFailure(msg, t);
}
});
} else if (DataConstants.ATTRIBUTES_DELETED.equals(msg.getType())) {
if (DataConstants.ATTRIBUTES_DELETED.equals(msg.getType())) {
List<String> attributes = new ArrayList<>();
for (JsonElement element : new JsonParser().parse(msg.getData()).getAsJsonObject().get("attributes").getAsJsonArray()) {
if (element.isJsonPrimitive()) {
@ -120,9 +104,15 @@ public class TbCopyAttributesToEntityViewNode implements TbNode {
List<String> filteredAttributes =
attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr, entityView)).collect(Collectors.toList());
if (!filteredAttributes.isEmpty()) {
ctx.getAttributesService().removeAll(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes);
transformAndTellNext(ctx, msg, entityView);
ctx.getTelemetryService().deleteAndNotify(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes,
getFutureCallback(ctx, msg, entityView));
}
} else {
Set<AttributeKvEntry> attributes = JsonConverter.convertToAttributes(new JsonParser().parse(msg.getData()));
List<AttributeKvEntry> filteredAttributes =
attributes.stream().filter(attr -> attributeContainsInEntityView(scope, attr.getKey(), entityView)).collect(Collectors.toList());
ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), entityView.getId(), scope, filteredAttributes,
getFutureCallback(ctx, msg, entityView));
}
}
}
@ -137,6 +127,21 @@ public class TbCopyAttributesToEntityViewNode implements TbNode {
}
}
@NotNull
private FutureCallback<Void> getFutureCallback(TbContext ctx, TbMsg msg, EntityView entityView) {
return new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void result) {
transformAndTellNext(ctx, msg, entityView);
}
@Override
public void onFailure(Throwable t) {
ctx.tellFailure(msg, t);
}
};
}
private void transformAndTellNext(TbContext ctx, TbMsg msg, EntityView entityView) {
ctx.enqueueForTellNext(ctx.newMsg(msg.getQueueName(), msg.getType(), entityView.getId(), msg.getCustomerId(), msg.getMetaData(), msg.getData()), SUCCESS);
}

10
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNode.java

@ -210,9 +210,13 @@ public class TbDeviceProfileNode implements TbNode {
DeviceState deviceState = deviceStates.get(deviceId);
if (deviceState != null) {
DeviceProfileId currentProfileId = deviceState.getProfileId();
Device device = JacksonUtil.fromString(deviceJson, Device.class);
if (!currentProfileId.equals(device.getDeviceProfileId())) {
removeDeviceState(deviceId);
try {
Device device = JacksonUtil.fromString(deviceJson, Device.class);
if (!currentProfileId.equals(device.getDeviceProfileId())) {
removeDeviceState(deviceId);
}
} catch (IllegalArgumentException e) {
log.debug("[{}] Received device update notification with non-device msg body: [{}][{}]", ctx.getSelfId(), deviceId, e);
}
}
}

45
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/AttributesDeleteNodeCallback.java

@ -0,0 +1,45 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.telemetry;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.server.common.msg.TbMsg;
import javax.annotation.Nullable;
import java.util.List;
@Slf4j
public class AttributesDeleteNodeCallback extends TelemetryNodeCallback {
private String scope;
private List<String> keys;
public AttributesDeleteNodeCallback(TbContext ctx, TbMsg msg, String scope, List<String> keys) {
super(ctx, msg);
this.scope = scope;
this.keys = keys;
}
@Override
public void onSuccess(@Nullable Void result) {
TbContext ctx = this.getCtx();
TbMsg tbMsg = this.getMsg();
ctx.enqueue(ctx.attributesDeletedActionMsg(tbMsg.getOriginator(), ctx.getSelfId(), scope, keys),
() -> ctx.tellSuccess(tbMsg),
throwable -> ctx.tellFailure(tbMsg, throwable));
}
}

44
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/AttributesUpdateNodeCallback.java

@ -0,0 +1,44 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.telemetry;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.msg.TbMsg;
import javax.annotation.Nullable;
import java.util.List;
public class AttributesUpdateNodeCallback extends TelemetryNodeCallback {
private String scope;
private List<AttributeKvEntry> attributes;
public AttributesUpdateNodeCallback(TbContext ctx, TbMsg msg, String scope, List<AttributeKvEntry> attributes) {
super(ctx, msg);
this.scope = scope;
this.attributes = attributes;
}
@Override
public void onSuccess(@Nullable Void result) {
TbContext ctx = this.getCtx();
TbMsg tbMsg = this.getMsg();
ctx.enqueue(ctx.attributesUpdatedActionMsg(tbMsg.getOriginator(), ctx.getSelfId(), scope, attributes),
() -> ctx.tellSuccess(tbMsg),
throwable -> ctx.tellFailure(tbMsg, throwable));
}
}

16
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.msg.session.SessionMsgType;
import org.thingsboard.server.common.transport.adaptor.JsonConverter;
import java.util.ArrayList;
import java.util.Set;
import java.util.List;
@Slf4j
@RuleNode(
@ -39,7 +39,9 @@ import java.util.Set;
name = "save attributes",
configClazz = TbMsgAttributesNodeConfiguration.class,
nodeDescription = "Saves attributes data",
nodeDetails = "Saves entity attributes based on configurable scope parameter. Expects messages with 'POST_ATTRIBUTES_REQUEST' message type",
nodeDetails = "Saves entity attributes based on configurable scope parameter. Expects messages with 'POST_ATTRIBUTES_REQUEST' message type. " +
"If upsert(update/insert) operation is completed successfully, rule node will send the \"Attributes Updated\" " +
"event to the root chain of the message originator and send the incoming message via <b>Success</b> chain, otherwise, <b>Failure</b> chain is used.",
uiResources = {"static/rulenode/rulenode-core-config.js"},
configDirective = "tbActionNodeAttributesConfig",
icon = "file_upload"
@ -63,15 +65,19 @@ public class TbMsgAttributesNode implements TbNode {
return;
}
String src = msg.getData();
Set<AttributeKvEntry> attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src));
List<AttributeKvEntry> attributes = new ArrayList<>(JsonConverter.convertToAttributes(new JsonParser().parse(src)));
if (attributes.isEmpty()) {
ctx.tellSuccess(msg);
return;
}
String notifyDeviceStr = msg.getMetaData().getValue("notifyDevice");
ctx.getTelemetryService().saveAndNotify(
ctx.getTenantId(),
msg.getOriginator(),
config.getScope(),
new ArrayList<>(attributes),
attributes,
config.getNotifyDevice() || StringUtils.isEmpty(notifyDeviceStr) || Boolean.parseBoolean(notifyDeviceStr),
new TelemetryNodeCallback(ctx, msg)
new AttributesUpdateNodeCallback(ctx, msg, config.getScope(), attributes)
);
}

73
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributes.java

@ -0,0 +1,73 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.telemetry;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.rule.engine.api.RuleNode;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNode;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.rule.engine.api.util.TbNodeUtils;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.msg.TbMsg;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@Slf4j
@RuleNode(
type = ComponentType.ACTION,
name = "delete attributes",
configClazz = TbMsgDeleteAttributesConfiguration.class,
nodeDescription = "Delete attributes for Message Originator.",
nodeDetails = "Attempt to remove attributes by selected keys. If msg originator doesn't have an attribute with " +
" a key selected in the configuration, it will be ignored. If delete operation is completed successfully, " +
" rule node will send the \"Attributes Deleted\" event to the root chain of the message originator and " +
" send the incoming message via <b>Success</b> chain, otherwise, <b>Failure</b> chain is used.",
uiResources = {"static/rulenode/rulenode-core-config.js"},
configDirective = "tbActionNodeDeleteAttributesConfig",
icon = "remove_circle"
)
public class TbMsgDeleteAttributes implements TbNode {
private TbMsgDeleteAttributesConfiguration config;
private String scope;
private List<String> keys;
@Override
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
this.config = TbNodeUtils.convert(configuration, TbMsgDeleteAttributesConfiguration.class);
this.scope = config.getScope();
this.keys = config.getKeys();
}
@Override
public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException {
List<String> keysToDelete = keys.stream()
.map(keyPattern -> TbNodeUtils.processPattern(keyPattern, msg))
.distinct()
.filter(StringUtils::isNotBlank)
.collect(Collectors.toList());
if (keysToDelete.isEmpty()) {
ctx.tellSuccess(msg);
} else {
ctx.getTelemetryService().deleteAndNotify(ctx.getTenantId(), msg.getOriginator(), scope, keysToDelete, new AttributesDeleteNodeCallback(ctx, msg, scope, keysToDelete));
}
}
}

38
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesConfiguration.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.telemetry;
import lombok.Data;
import org.thingsboard.rule.engine.api.NodeConfiguration;
import org.thingsboard.server.common.data.DataConstants;
import java.util.Collections;
import java.util.List;
@Data
public class TbMsgDeleteAttributesConfiguration implements NodeConfiguration<TbMsgDeleteAttributesConfiguration> {
private String scope;
private List<String> keys;
@Override
public TbMsgDeleteAttributesConfiguration defaultConfiguration() {
TbMsgDeleteAttributesConfiguration configuration = new TbMsgDeleteAttributesConfiguration();
configuration.setScope(DataConstants.SERVER_SCOPE);
configuration.setKeys(Collections.emptyList());
return configuration;
}
}

125
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgDeleteAttributesTest.java

@ -0,0 +1,125 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.telemetry;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.junit.MockitoJUnitRunner;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Consumer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.BDDMockito.willAnswer;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@Slf4j
public class TbMsgDeleteAttributesTest {
final ObjectMapper mapper = new ObjectMapper();
DeviceId deviceId;
TbMsgDeleteAttributes node;
TbMsgDeleteAttributesConfiguration config;
TbNodeConfiguration nodeConfiguration;
TbContext ctx;
TbMsgCallback callback;
RuleEngineTelemetryService telemetryService;
@BeforeEach
void setUp() throws TbNodeException {
deviceId = new DeviceId(UUID.randomUUID());
callback = mock(TbMsgCallback.class);
ctx = mock(TbContext.class);
config = new TbMsgDeleteAttributesConfiguration().defaultConfiguration();
config.setKeys(List.of("${TestAttribute_1}", "$[TestAttribute_2]", "$[TestAttribute_3]", "TestAttribute_4"));
nodeConfiguration = new TbNodeConfiguration(mapper.valueToTree(config));
node = spy(new TbMsgDeleteAttributes());
node.init(ctx, nodeConfiguration);
telemetryService = mock(RuleEngineTelemetryService.class);
willReturn(telemetryService).given(ctx).getTelemetryService();
willAnswer(invocation -> {
TelemetryNodeCallback callBack = invocation.getArgument(4);
callBack.onSuccess(null);
return null;
}).given(telemetryService).deleteAndNotify(
any(), any(), anyString(), anyList(), any());
}
@AfterEach
void tearDown() {
node.destroy();
}
@Test
void givenDefaultConfig_whenVerify_thenOK() {
TbMsgDeleteAttributesConfiguration defaultConfig = new TbMsgDeleteAttributesConfiguration().defaultConfiguration();
assertThat(defaultConfig.getScope()).isEqualTo(DataConstants.SERVER_SCOPE);
assertThat(defaultConfig.getKeys()).isEqualTo(Collections.emptyList());
}
@Test
void givenMsg_whenOnMsg_thenVerifyOutput() throws Exception {
final Map<String, String> mdMap = Map.of(
"TestAttribute_1", "temperature",
"city", "NY"
);
final TbMsgMetaData metaData = new TbMsgMetaData(mdMap);
final String data = "{\"TestAttribute_2\": \"humidity\", \"TestAttribute_3\": \"voltage\"}";
TbMsg msg = TbMsg.newMsg("POST_ATTRIBUTES_REQUEST", deviceId, metaData, data, callback);
node.onMsg(ctx, msg);
ArgumentCaptor<Runnable> successCaptor = ArgumentCaptor.forClass(Runnable.class);
ArgumentCaptor<Consumer<Throwable>> failureCaptor = ArgumentCaptor.forClass(Consumer.class);
ArgumentCaptor<TbMsg> newMsgCaptor = ArgumentCaptor.forClass(TbMsg.class);
verify(ctx, times(1)).enqueue(any(), successCaptor.capture(), failureCaptor.capture());
successCaptor.getValue().run();
verify(ctx, times(1)).tellSuccess(newMsgCaptor.capture());
verify(ctx, times(1)).attributesDeletedActionMsg(any(), any(), anyString(), anyList());
verify(ctx, never()).tellFailure(any(), any());
verify(telemetryService, times(1)).deleteAndNotify(any(), any(), anyString(), anyList(), any());
}
}
Loading…
Cancel
Save