tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #15917 from ShvaykaD/fix/entity-aggregation-cf-numeric-result 

Fixed timeseries entity aggregation calculated field storing numeric results as strings
pull/16018/merge
Viacheslav Klimov 7 days ago
committed by GitHub
parent
52feea3326 da3f2e933a
commit
567a48e429
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
3 changed files with 216 additions and 17 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java
  2. 45
      application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java
  3. 186
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldStateTest.java

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldState.java
View File

@ -293,7 +293,7 @@ public class EntityAggregationCalculatedFieldState extends BaseCalculatedFieldSt
Object resultValue = argumentEntry.getValue() instanceof Number number
? NumberUtils.roundResult(number.doubleValue(), precision)
: argumentEntry.getValue();
metricsNode.put(metricName, JacksonUtil.toString(resultValue));
metricsNode.set(metricName, JacksonUtil.valueToTree(resultValue));
}
}
if (!metricsNode.isEmpty()) {

45
application/src/test/java/org/thingsboard/server/cf/EntityAggregationCalculatedFieldTest.java
View File

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.cf;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.After;
import org.junit.Before;
@ -106,7 +107,7 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgConsumption");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("9999");
assertNumericValue(result, "consumption", 9999);
assertThat(result.get("avgConsumption").get(0).get("value").isNull()).isTrue();
});
}
@ -137,8 +138,8 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgConsumption");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400");
assertThat(result.get("avgConsumption").get(0).get("value").asText()).isEqualTo("133");
assertNumericValue(result, "consumption", 400);
assertNumericValue(result, "avgConsumption", 133);
});
postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":500}}", tsInInterval_1));
@ -149,8 +150,8 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgConsumption");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400");
assertThat(result.get("avgConsumption").get(0).get("value").asText()).isEqualTo("133");
assertNumericValue(result, "consumption", 400);
assertNumericValue(result, "avgConsumption", 133);
});
}
@ -181,8 +182,8 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgConsumption");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400");
assertThat(result.get("avgConsumption").get(0).get("value").asText()).isEqualTo("133");
assertNumericValue(result, "consumption", 400);
assertNumericValue(result, "avgConsumption", 133);
});
postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":300}}", tsInInterval_1));
@ -193,8 +194,8 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgConsumption");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("600");
assertThat(result.get("avgConsumption").get(0).get("value").asText()).isEqualTo("200");
assertNumericValue(result, "consumption", 600);
assertNumericValue(result, "avgConsumption", 200);
});
}
@ -231,8 +232,8 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgConsumption");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400");
assertThat(result.get("avgConsumption").get(0).get("value").asText()).isEqualTo("133");
assertNumericValue(result, "consumption", 400);
assertNumericValue(result, "avgConsumption", 133);
});
postTelemetry(device.getId(), String.format("{\"ts\": \"%s\", \"values\": {\"energy\":500}}", currentIntervalStartTs + 4500L));
@ -243,9 +244,9 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgConsumption");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("500");
assertNumericValue(result, "consumption", 500);
assertThat(result.get("consumption").get(0).get("ts").asLong()).isEqualTo(currentIntervalStartTs + 4000L);
assertThat(result.get("avgConsumption").get(0).get("value").asText()).isEqualTo("500");
assertNumericValue(result, "avgConsumption", 500);
assertThat(result.get("avgConsumption").get(0).get("ts").asLong()).isEqualTo(currentIntervalStartTs + 4000L);
});
}
@ -306,8 +307,8 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
.untilAsserted(() -> {
ObjectNode result = getLatestTelemetry(device.getId(), "consumption", "avgTemperature");
assertThat(result).isNotNull();
assertThat(result.get("consumption").get(0).get("value").asText()).isEqualTo("400");
assertThat(result.get("avgTemperature").get(0).get("value").asText()).isEqualTo("39");
assertNumericValue(result, "consumption", 400);
assertNumericValue(result, "avgTemperature", 39);
});
}
@ -372,8 +373,20 @@ public class EntityAggregationCalculatedFieldTest extends AbstractControllerTest
return saveCalculatedField(calculatedField);
}
// useStrictDataTypes=true so the value node keeps its stored type (numeric -> JSON number, str_v -> JSON string).
// Without it the endpoint returns every value via getValueAsString(), masking the string-vs-number distinction.
private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception {
return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class);
return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?useStrictDataTypes=true&keys=" + String.join(",", keys), ObjectNode.class);
}
// Regression guard: a numeric aggregation result must be stored as a numeric JSON node (ts_kv.dbl_v/long_v),
// not a JSON string (ts_kv.str_v) - otherwise server-side AVG/SUM return no data. A value-only check would
// not catch this: asLong()/asText() coerce a string node like "400" to the same value/text, so the node type
// is asserted explicitly; the numeric comparison then verifies the aggregated value.
private static void assertNumericValue(ObjectNode result, String key, long expectedValue) {
JsonNode value = result.get(key).get(0).get("value");
assertThat(value.isNumber()).as(key + " should be stored as a numeric node").isTrue();
assertThat(value.asLong()).isEqualTo(expectedValue);
}
}

186
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/aggregation/single/EntityAggregationCalculatedFieldStateTest.java
View File

@ -0,0 +1,186 @@
/**
* Copyright © 2016-2026 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.cf.ctx.state.aggregation.single;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.TimeSeriesOutput;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric;
import org.thingsboard.server.common.data.cf.configuration.aggregation.single.EntityAggregationCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.aggregation.single.interval.CustomInterval;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
public class EntityAggregationCalculatedFieldStateTest {
private static final long INTERVAL_START_TS = 1_000L;
private static final long INTERVAL_END_TS = 2_000L;
private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("80ee80ef-019f-46b1-80ba-22f3ef1b094c"));
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("fc83d188-9cf5-4919-a774-d5c56bba2d27"));
private EntityAggregationCalculatedFieldState state;
private CalculatedFieldCtx ctx;
@Mock
private TenantProfile tenantProfile;
@Mock
private TbTenantProfileCache tenantProfileCache;
@InjectMocks
private ActorSystemContext systemContext;
@BeforeEach
void setUp() {
when(tenantProfileCache.get(any(TenantId.class))).thenReturn(tenantProfile);
when(tenantProfile.getProfileConfiguration()).thenReturn(Optional.of(new DefaultTenantProfileConfiguration()));
ctx = new CalculatedFieldCtx(getCalculatedField(), systemContext);
ctx.init();
state = new EntityAggregationCalculatedFieldState(DEVICE_ID);
state.setCtx(ctx, null);
state.init(false);
}
@Test
void testType() {
assertThat(state.getType()).isEqualTo(CalculatedFieldType.ENTITY_AGGREGATION);
}
// A numeric aggregation result (SUM/AVG/COUNT/..., numeric MIN/MAX) must be serialized as a numeric
// JSON node; a genuine string result (lexical MIN/MAX over string telemetry, e.g. a zero-padded code)
// must stay a string node. The node type is asserted explicitly, so asText() is only used to verify the
// value once the type is already pinned - it is not relied on to distinguish the types (that blindness
// is what hid the bug).
@ParameterizedTest(name = "{0} (precision {2}) -> numeric={3}")
@MethodSource("toResultSerializationCases")
void toResultSerializesResultWithTypePreservingNode(String metricName, BasicKvEntry kvEntry, Integer precision,
boolean expectNumeric, String expectedText) {
JsonNode value = toResultValue(metricName, kvEntry, precision);
assertThat(value.isNumber()).isEqualTo(expectNumeric);
assertThat(value.isTextual()).isEqualTo(!expectNumeric);
assertThat(value.asText()).isEqualTo(expectedText);
}
private static Stream<Arguments> toResultSerializationCases() {
return Stream.of(
// SUM: Number result, precision 0 -> whole-number (long) node
arguments("consumption", new DoubleDataEntry("consumption", 400.0), 0, true, "400"),
// AVG: Number result, precision 2 -> half-up rounded double node
arguments("avgConsumption", new DoubleDataEntry("avgConsumption", 133.335), 2, true, "133.34"),
// MAX over string telemetry: a zero-padded code is a genuine String result and must stay a
// string node - as a number it would lose its padding ("0009" -> 9).
arguments("maxCode", new StringDataEntry("maxCode", "0009"), 0, false, "0009")
);
}
private JsonNode toResultValue(String metricName, BasicKvEntry kvEntry, Integer precision) {
AggIntervalEntry interval = new AggIntervalEntry(INTERVAL_START_TS, INTERVAL_END_TS);
ArgumentEntry argumentEntry = new SingleValueArgumentEntry(INTERVAL_START_TS, kvEntry, SingleValueArgumentEntry.DEFAULT_VERSION);
Map<AggIntervalEntry, Map<String, ArgumentEntry>> results = new HashMap<>();
results.put(interval, Map.of(metricName, argumentEntry));
ArrayNode result = state.toResult(results, precision);
assertThat(result.size()).isEqualTo(1);
assertThat(result.get(0).get("ts").asLong()).isEqualTo(INTERVAL_START_TS);
return result.get(0).get("values").get(metricName);
}
private CalculatedField getCalculatedField() {
CalculatedField calculatedField = new CalculatedField();
calculatedField.setTenantId(TENANT_ID);
calculatedField.setEntityId(DEVICE_ID);
calculatedField.setType(CalculatedFieldType.ENTITY_AGGREGATION);
calculatedField.setName("Test Entity Aggregation CF");
calculatedField.setConfigurationVersion(1);
calculatedField.setConfiguration(getConfiguration());
calculatedField.setVersion(1L);
return calculatedField;
}
private EntityAggregationCalculatedFieldConfiguration getConfiguration() {
EntityAggregationCalculatedFieldConfiguration configuration = new EntityAggregationCalculatedFieldConfiguration();
Argument energy = new Argument();
energy.setRefEntityKey(new ReferencedEntityKey("energy", ArgumentType.TS_LATEST, null));
configuration.setArguments(Map.of("en", energy));
Map<String, AggMetric> metrics = new HashMap<>();
AggMetric consumption = new AggMetric();
consumption.setFunction(AggFunction.SUM);
consumption.setInput(new AggKeyInput("en"));
metrics.put("consumption", consumption);
AggMetric avgConsumption = new AggMetric();
avgConsumption.setFunction(AggFunction.AVG);
avgConsumption.setInput(new AggKeyInput("en"));
metrics.put("avgConsumption", avgConsumption);
AggMetric maxCode = new AggMetric();
maxCode.setFunction(AggFunction.MAX);
maxCode.setInput(new AggKeyInput("en"));
metrics.put("maxCode", maxCode);
configuration.setMetrics(metrics);
configuration.setInterval(new CustomInterval("UTC", 0L, 5L));
TimeSeriesOutput output = new TimeSeriesOutput();
output.setDecimalsByDefault(0);
configuration.setOutput(output);
return configuration;
}
}
Write Preview
Loading…
Cancel
Save