From 2b14fa3634cd55fcef6bc71b83bea1bbcbec16f0 Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Tue, 14 Jul 2020 15:38:21 +0300 Subject: [PATCH 1/5] fixed bug with saving server attributes in client scope --- .../edge/rpc/init/DefaultSyncEdgeService.java | 20 ++++++++++++------- common/queue/src/main/proto/queue.proto | 1 + .../transport/adaptor/JsonConverter.java | 3 ++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/init/DefaultSyncEdgeService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/init/DefaultSyncEdgeService.java index 8576f96152..1a2a3d3177 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/init/DefaultSyncEdgeService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/init/DefaultSyncEdgeService.java @@ -71,7 +71,9 @@ import org.thingsboard.server.gen.edge.UserCredentialsRequestMsg; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; @Service @@ -289,25 +291,29 @@ public class DefaultSyncEdgeService implements SyncEdgeService { public void onSuccess(@Nullable List ssAttributes) { if (ssAttributes != null && !ssAttributes.isEmpty()) { try { - ObjectNode entityNode = mapper.createObjectNode(); + Map entityData = new HashMap<>(); + ObjectNode attributes = mapper.createObjectNode(); for (AttributeKvEntry attr : ssAttributes) { if (attr.getDataType() == DataType.BOOLEAN && attr.getBooleanValue().isPresent()) { - entityNode.put(attr.getKey(), attr.getBooleanValue().get()); + attributes.put(attr.getKey(), attr.getBooleanValue().get()); } else if (attr.getDataType() == DataType.DOUBLE && attr.getDoubleValue().isPresent()) { - entityNode.put(attr.getKey(), attr.getDoubleValue().get()); + attributes.put(attr.getKey(), attr.getDoubleValue().get()); } else if (attr.getDataType() == DataType.LONG && attr.getLongValue().isPresent()) { - entityNode.put(attr.getKey(), attr.getLongValue().get()); + attributes.put(attr.getKey(), attr.getLongValue().get()); } else { - entityNode.put(attr.getKey(), attr.getValueAsString()); + attributes.put(attr.getKey(), attr.getValueAsString()); } } - log.debug("Sending attributes data msg, entityId [{}], attributes [{}]", entityId, entityNode); + entityData.put("kv", attributes); + entityData.put("scope", DataConstants.SERVER_SCOPE); + JsonNode entityBody = mapper.valueToTree(entityData); + log.debug("Sending attributes data msg, entityId [{}], attributes [{}]", entityId, entityBody); saveEdgeEvent(edge.getTenantId(), edge.getId(), edgeEventType, ActionType.ATTRIBUTES_UPDATED, entityId, - entityNode); + entityBody); } catch (Exception e) { log.error("[{}] Failed to send attribute updates to the edge", edge.getName(), e); } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 921a096df0..c302a30b8c 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -115,6 +115,7 @@ message PostTelemetryMsg { message PostAttributeMsg { repeated KeyValueProto kv = 1; + string scope = 2; } message GetAttributeRequestMsg { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 8375b84ffa..2836f05164 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -136,8 +136,9 @@ public class JsonConverter { public static PostAttributeMsg convertToAttributesProto(JsonElement jsonObject) throws JsonSyntaxException { if (jsonObject.isJsonObject()) { PostAttributeMsg.Builder result = PostAttributeMsg.newBuilder(); - List keyValueList = parseProtoValues(jsonObject.getAsJsonObject()); + List keyValueList = parseProtoValues(jsonObject.getAsJsonObject().getAsJsonObject("kv")); result.addAllKv(keyValueList); + result.setScope(jsonObject.getAsJsonObject().getAsJsonPrimitive("scope").getAsString()); return result.build(); } else { throw new JsonSyntaxException(CAN_T_PARSE_VALUE + jsonObject); From 37cb35c99baba90b72c313aa9e6681c8a293bce9 Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Tue, 14 Jul 2020 17:06:21 +0300 Subject: [PATCH 2/5] fix + refactoring --- .../common/transport/adaptor/JsonConverter.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 2836f05164..42b4b2a77c 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -133,15 +133,21 @@ public class JsonConverter { .build(); } - public static PostAttributeMsg convertToAttributesProto(JsonElement jsonObject) throws JsonSyntaxException { - if (jsonObject.isJsonObject()) { + public static PostAttributeMsg convertToAttributesProto(JsonElement jsonElement) throws JsonSyntaxException { + if (jsonElement.isJsonObject()) { PostAttributeMsg.Builder result = PostAttributeMsg.newBuilder(); - List keyValueList = parseProtoValues(jsonObject.getAsJsonObject().getAsJsonObject("kv")); + List keyValueList = null; + JsonObject jsonObject = jsonElement.getAsJsonObject(); + if (jsonObject.has("kv") && jsonObject.has("scope")) { + keyValueList = parseProtoValues(jsonObject.getAsJsonObject("kv")); + result.setScope(jsonObject.getAsJsonPrimitive("scope").getAsString()); + } else { + keyValueList = parseProtoValues(jsonObject); + } result.addAllKv(keyValueList); - result.setScope(jsonObject.getAsJsonObject().getAsJsonPrimitive("scope").getAsString()); return result.build(); } else { - throw new JsonSyntaxException(CAN_T_PARSE_VALUE + jsonObject); + throw new JsonSyntaxException(CAN_T_PARSE_VALUE + jsonElement); } } From d81804ac8ddc517f15d94be393316e525dac0efc Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Tue, 14 Jul 2020 19:22:31 +0300 Subject: [PATCH 3/5] scope replaced to EntityDataProto --- .../rpc/constructor/EntityDataMsgConstructor.java | 9 ++++++++- common/edge-api/src/main/proto/edge.proto | 1 + common/queue/src/main/proto/queue.proto | 1 - .../common/transport/adaptor/JsonConverter.java | 15 ++++----------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java index d9a06655c5..e83c197690 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.edge.rpc.constructor; import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.audit.ActionType; @@ -42,7 +43,13 @@ public class EntityDataMsgConstructor { break; case ATTRIBUTES_UPDATED: try { - builder.setPostAttributesMsg(JsonConverter.convertToAttributesProto(entityData)); + JsonObject data = entityData.getAsJsonObject(); + if (data.has("scope") && data.has("kv")) { + builder.setPostAttributesMsg(JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv"))); + builder.setPostAttributeScope(data.getAsJsonPrimitive("scope").getAsString()); + } else { + builder.setPostAttributesMsg(JsonConverter.convertToAttributesProto(data)); + } } catch (Exception e) { log.warn("Can't convert to attributes proto, entityData [{}]", entityData, e); } diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 2cdbd241d1..9713cd754f 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -109,6 +109,7 @@ message EntityDataProto { string entityType = 3; transport.PostTelemetryMsg postTelemetryMsg = 4; transport.PostAttributeMsg postAttributesMsg = 5; + string postAttributeScope = 6; // transport.ToDeviceRpcRequestMsg ??? } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index c302a30b8c..921a096df0 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -115,7 +115,6 @@ message PostTelemetryMsg { message PostAttributeMsg { repeated KeyValueProto kv = 1; - string scope = 2; } message GetAttributeRequestMsg { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 42b4b2a77c..8375b84ffa 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -133,21 +133,14 @@ public class JsonConverter { .build(); } - public static PostAttributeMsg convertToAttributesProto(JsonElement jsonElement) throws JsonSyntaxException { - if (jsonElement.isJsonObject()) { + public static PostAttributeMsg convertToAttributesProto(JsonElement jsonObject) throws JsonSyntaxException { + if (jsonObject.isJsonObject()) { PostAttributeMsg.Builder result = PostAttributeMsg.newBuilder(); - List keyValueList = null; - JsonObject jsonObject = jsonElement.getAsJsonObject(); - if (jsonObject.has("kv") && jsonObject.has("scope")) { - keyValueList = parseProtoValues(jsonObject.getAsJsonObject("kv")); - result.setScope(jsonObject.getAsJsonPrimitive("scope").getAsString()); - } else { - keyValueList = parseProtoValues(jsonObject); - } + List keyValueList = parseProtoValues(jsonObject.getAsJsonObject()); result.addAllKv(keyValueList); return result.build(); } else { - throw new JsonSyntaxException(CAN_T_PARSE_VALUE + jsonElement); + throw new JsonSyntaxException(CAN_T_PARSE_VALUE + jsonObject); } } From 6ede96ef48531ffec5fca2c3fd26b5ae47cf8ff2 Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Wed, 15 Jul 2020 14:54:25 +0300 Subject: [PATCH 4/5] fix for POST_ATTRIBUTES_REQUEST --- .../rpc/constructor/EntityDataMsgConstructor.java | 8 ++------ .../rule/engine/edge/TbMsgPushToEdgeNode.java | 14 +++++++++++++- .../rule/engine/telemetry/TbMsgAttributesNode.java | 1 + 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java index e83c197690..2c4694d25d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java @@ -44,12 +44,8 @@ public class EntityDataMsgConstructor { case ATTRIBUTES_UPDATED: try { JsonObject data = entityData.getAsJsonObject(); - if (data.has("scope") && data.has("kv")) { - builder.setPostAttributesMsg(JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv"))); - builder.setPostAttributeScope(data.getAsJsonPrimitive("scope").getAsString()); - } else { - builder.setPostAttributesMsg(JsonConverter.convertToAttributesProto(data)); - } + builder.setPostAttributesMsg(JsonConverter.convertToAttributesProto(data.getAsJsonObject("kv"))); + builder.setPostAttributeScope(data.getAsJsonPrimitive("scope").getAsString()); } catch (Exception e) { log.warn("Can't convert to attributes proto, entityData [{}]", entityData, e); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java index eef30d5d22..b383ae0a1a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java @@ -46,7 +46,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.session.SessionMsgType; import javax.annotation.Nullable; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import static org.thingsboard.rule.engine.api.TbRelationTypes.SUCCESS; @@ -137,7 +139,17 @@ public class TbMsgPushToEdgeNode implements TbNode { if (edgeEventTypeByEntityType == null) { return null; } - return buildEdgeEvent(ctx.getTenantId(), getActionTypeByMsgType(msg.getType()), msg.getOriginator().getId(), edgeEventTypeByEntityType, json.readTree(msg.getData())); + JsonNode entityBody = null; + JsonNode data = json.readTree(msg.getData()); + if (SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { + Map entityData = new HashMap<>(); + entityData.put("kv", data); + entityData.put("scope", msg.getMetaData().getData().get("scope")); + entityBody = json.valueToTree(entityData); + } else { + entityBody = data; + } + return buildEdgeEvent(ctx.getTenantId(), getActionTypeByMsgType(msg.getType()), msg.getOriginator().getId(), edgeEventTypeByEntityType, entityBody); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index e776fa4dbd..b6e9e4a292 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -64,6 +64,7 @@ public class TbMsgAttributesNode implements TbNode { } String src = msg.getData(); Set attributes = JsonConverter.convertToAttributes(new JsonParser().parse(src)); + msg.getMetaData().putValue("scope", config.getScope()); ctx.getTelemetryService().saveAndNotify(ctx.getTenantId(), msg.getOriginator(), config.getScope(), new ArrayList<>(attributes), new TelemetryNodeCallback(ctx, msg)); } From e571bf24b6ad2f8b1bdee0a728adf90a54e9a41e Mon Sep 17 00:00:00 2001 From: Bohdan Smetaniuk Date: Wed, 15 Jul 2020 19:37:01 +0300 Subject: [PATCH 5/5] complited attributes update and attributes delete feature --- .../constructor/EntityDataMsgConstructor.java | 20 +++++++++++++-- common/edge-api/src/main/proto/edge.proto | 6 +++++ .../rule/engine/edge/TbMsgPushToEdgeNode.java | 25 ++++++++++++++----- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java index 2c4694d25d..07f7ffb02a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.edge.rpc.constructor; +import com.google.gson.Gson; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; @@ -22,8 +24,11 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.gen.edge.AttributeDeleteMsg; import org.thingsboard.server.gen.edge.EntityDataProto; +import java.util.List; + @Component @Slf4j public class EntityDataMsgConstructor { @@ -50,8 +55,19 @@ public class EntityDataMsgConstructor { log.warn("Can't convert to attributes proto, entityData [{}]", entityData, e); } break; - // TODO: voba - add support for attribute delete - // case ATTRIBUTES_DELETED: + case ATTRIBUTES_DELETED: + try { + AttributeDeleteMsg.Builder attributeDeleteMsg = AttributeDeleteMsg.newBuilder(); + attributeDeleteMsg.setScope(entityData.getAsJsonObject().getAsJsonPrimitive("scope").getAsString()); + JsonArray jsonArray = entityData.getAsJsonObject().getAsJsonArray("keys"); + List keys = new Gson().fromJson(jsonArray.toString(), List.class); + attributeDeleteMsg.addAllAttributeNames(keys); + attributeDeleteMsg.build(); + builder.setAttributeDeleteMsg(attributeDeleteMsg); + } catch (Exception e) { + log.warn("Can't convert to AttributeDeleteMsg proto, entityData [{}]", entityData, e); + } + break; } return builder.build(); } diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 9713cd754f..181ebff531 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -110,9 +110,15 @@ message EntityDataProto { transport.PostTelemetryMsg postTelemetryMsg = 4; transport.PostAttributeMsg postAttributesMsg = 5; string postAttributeScope = 6; + AttributeDeleteMsg attributeDeleteMsg = 7; // transport.ToDeviceRpcRequestMsg ??? } +message AttributeDeleteMsg { + string scope = 1; + repeated string attributeNames = 2; +} + message RuleChainUpdateMsg { UpdateMsgType msgType = 1; int64 idMSB = 2; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java index b383ae0a1a..f1134078c1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNode.java @@ -139,17 +139,15 @@ public class TbMsgPushToEdgeNode implements TbNode { if (edgeEventTypeByEntityType == null) { return null; } + ActionType actionType = getActionTypeByMsgType(msg.getType()); JsonNode entityBody = null; JsonNode data = json.readTree(msg.getData()); - if (SessionMsgType.POST_ATTRIBUTES_REQUEST.name().equals(msg.getType())) { - Map entityData = new HashMap<>(); - entityData.put("kv", data); - entityData.put("scope", msg.getMetaData().getData().get("scope")); - entityBody = json.valueToTree(entityData); + if (actionType.equals(ActionType.ATTRIBUTES_UPDATED) || actionType.equals(ActionType.ATTRIBUTES_DELETED)) { + entityBody = getAttributeEntityBody(actionType, data, msg.getMetaData().getData()); } else { entityBody = data; } - return buildEdgeEvent(ctx.getTenantId(), getActionTypeByMsgType(msg.getType()), msg.getOriginator().getId(), edgeEventTypeByEntityType, entityBody); + return buildEdgeEvent(ctx.getTenantId(), actionType, msg.getOriginator().getId(), edgeEventTypeByEntityType, entityBody); } } @@ -163,6 +161,21 @@ public class TbMsgPushToEdgeNode implements TbNode { return edgeEvent; } + private JsonNode getAttributeEntityBody(ActionType actionType, JsonNode data, Map metadata) throws JsonProcessingException { + Map entityData = new HashMap<>(); + switch (actionType) { + case ATTRIBUTES_UPDATED: + entityData.put("kv", data); + break; + case ATTRIBUTES_DELETED: + List keys = json.treeToValue(data.get("attributes"), List.class); + entityData.put("keys", keys); + break; + } + entityData.put("scope", metadata.get("scope")); + return json.valueToTree(entityData); + } + private UUID getUUIDFromMsgData(TbMsg msg) throws JsonProcessingException { JsonNode data = json.readTree(msg.getData()).get("id"); String id = json.treeToValue(data.get("id"), String.class);