From 9fe262fa4cf71c630c10ea20fd96a27e4e2b22cb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 25 Jun 2024 12:57:00 +0300 Subject: [PATCH 01/32] added tests for originator telemetry node --- .../engine/metadata/TbGetTelemetryNode.java | 3 +- .../metadata/TbGetTelemetryNodeTest.java | 341 ++++++++++++++++-- 2 files changed, 306 insertions(+), 38 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index bcf6b1b05a..86f309ab09 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -40,7 +40,6 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -91,7 +90,7 @@ public class TbGetTelemetryNode implements TbNode { } @Override - public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException { + public void onMsg(TbContext ctx, TbMsg msg) { if (tsKeyNames.isEmpty()) { ctx.tellFailure(msg, new IllegalStateException("Telemetry is not selected!")); } else { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java index 6ecd1b6eb9..1875dd91f1 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java @@ -15,72 +15,341 @@ */ package org.thingsboard.rule.engine.metadata; +import com.google.common.util.concurrent.Futures; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.rule.engine.TestDbCallbackExecutor; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.dao.timeseries.TimeseriesService; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.willCallRealMethod; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +@ExtendWith(MockitoExtension.class) public class TbGetTelemetryNodeTest { - TbGetTelemetryNode node; - TbGetTelemetryNodeConfiguration config; - TbNodeConfiguration nodeConfiguration; - TbContext ctx; + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5738401b-9dba-422b-b656-a62fe7431917")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("8a8fd749-b2ec-488b-a6c6-fc66614d8686")); + + private final ListeningExecutor executor = new TestDbCallbackExecutor(); + + private TbGetTelemetryNode node; + private TbGetTelemetryNodeConfiguration config; + + @Mock + private TbContext ctxMock; + @Mock + private TimeseriesService timeseriesServiceMock; @BeforeEach public void setUp() throws Exception { - ctx = mock(TbContext.class); - node = spy(new TbGetTelemetryNode()); - config = new TbGetTelemetryNodeConfiguration(); - config.setFetchMode("ALL"); - nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - node.init(ctx, nodeConfiguration); - - willCallRealMethod().given(node).parseAggregationConfig(any()); + node = new TbGetTelemetryNode(); + config = new TbGetTelemetryNodeConfiguration().defaultConfiguration(); } @Test - public void givenAggregationAsString_whenParseAggregation_thenReturnEnum() { + public void givenAggregationAsString_whenParseAggregation_thenReturnEnum() throws TbNodeException { + config.setFetchMode("ALL"); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + //compatibility with old configs without "aggregation" parameter - assertThat(node.parseAggregationConfig(null), is(Aggregation.NONE)); - assertThat(node.parseAggregationConfig(""), is(Aggregation.NONE)); + assertThat(node.parseAggregationConfig(null)).isEqualTo(Aggregation.NONE); + assertThat(node.parseAggregationConfig("")).isEqualTo(Aggregation.NONE); //common values - assertThat(node.parseAggregationConfig("MIN"), is(Aggregation.MIN)); - assertThat(node.parseAggregationConfig("MAX"), is(Aggregation.MAX)); - assertThat(node.parseAggregationConfig("AVG"), is(Aggregation.AVG)); - assertThat(node.parseAggregationConfig("SUM"), is(Aggregation.SUM)); - assertThat(node.parseAggregationConfig("COUNT"), is(Aggregation.COUNT)); - assertThat(node.parseAggregationConfig("NONE"), is(Aggregation.NONE)); + assertThat(node.parseAggregationConfig("MIN")).isEqualTo(Aggregation.MIN); + assertThat(node.parseAggregationConfig("MAX")).isEqualTo(Aggregation.MAX); + assertThat(node.parseAggregationConfig("AVG")).isEqualTo(Aggregation.AVG); + assertThat(node.parseAggregationConfig("SUM")).isEqualTo(Aggregation.SUM); + assertThat(node.parseAggregationConfig("COUNT")).isEqualTo(Aggregation.COUNT); + assertThat(node.parseAggregationConfig("NONE")).isEqualTo(Aggregation.NONE); //all possible values in future for (Aggregation aggEnum : Aggregation.values()) { - assertThat(node.parseAggregationConfig(aggEnum.name()), is(aggEnum)); + assertThat(node.parseAggregationConfig(aggEnum.name())).isEqualTo(aggEnum); } } @Test - public void givenAggregationWhiteSpace_whenParseAggregation_thenException() { - Assertions.assertThrows(IllegalArgumentException.class, () -> { - node.parseAggregationConfig(" "); - }); + public void givenAggregationWhiteSpace_whenParseAggregation_thenException() throws TbNodeException { + config.setFetchMode("ALL"); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + assertThatThrownBy(() -> node.parseAggregationConfig(" ")).isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void givenAggregationIncorrect_whenParseAggregation_thenException() throws TbNodeException { + config.setFetchMode("ALL"); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + assertThatThrownBy(() -> node.parseAggregationConfig("TOP")).isInstanceOf(IllegalArgumentException.class); } @Test - public void givenAggregationIncorrect_whenParseAggregation_thenException() { - Assertions.assertThrows(IllegalArgumentException.class, () -> { - node.parseAggregationConfig("TOP"); - }); + public void verifyDefaultConfig() { + assertThat(config.getStartInterval()).isEqualTo(2); + assertThat(config.getEndInterval()).isEqualTo(1); + assertThat(config.getStartIntervalPattern()).isEqualTo(""); + assertThat(config.getEndIntervalPattern()).isEqualTo(""); + assertThat(config.isUseMetadataIntervalPatterns()).isFalse(); + assertThat(config.getStartIntervalTimeUnit()).isEqualTo(TimeUnit.MINUTES.name()); + assertThat(config.getEndIntervalTimeUnit()).isEqualTo(TimeUnit.MINUTES.name()); + assertThat(config.getFetchMode()).isEqualTo("FIRST"); + assertThat(config.getOrderBy()).isEqualTo("ASC"); + assertThat(config.getAggregation()).isEqualTo(Aggregation.NONE.name()); + assertThat(config.getLimit()).isEqualTo(1000); + assertThat(config.getLatestTsKeyNames()).isEmpty(); } + @ParameterizedTest + @MethodSource + public void givenFetchModeAndLimit_whenInit_thenVerifyLimit(String fetchMode, int limit, int expectedLimit) throws TbNodeException { + config.setFetchMode(fetchMode); + config.setLimit(limit); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var actualLimit = ReflectionTestUtils.getField(node, "limit"); + assertThat(actualLimit).isEqualTo(expectedLimit); + } + + private static Stream givenFetchModeAndLimit_whenInit_thenVerifyLimit() { + return Stream.of( + Arguments.of("FIRST", 0, 1), + Arguments.of("LAST", 10, 1), + Arguments.of("ALL", 0, 1000), + Arguments.of("ALL", 5, 5) + ); + } + + @ParameterizedTest + @NullAndEmptySource + public void givenEmptyOrderBy_whenInit_thenVerify(String orderBy) throws TbNodeException { + config.setOrderBy(orderBy); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var actualOrderBy = ReflectionTestUtils.getField(node, "orderByFetchAll"); + assertThat(actualOrderBy).isEqualTo("ASC"); + } + + @Test + public void givenConfig_whenInit_thenVerify() throws TbNodeException { + List keys = List.of("temperature", "humidity"); + config.setLatestTsKeyNames(keys); + config.setFetchMode("ALL"); + config.setLimit(5); + config.setOrderBy("DESC"); + config.setAggregation("MIN"); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var actualLimit = ReflectionTestUtils.getField(node, "limit"); + var actualTsKeyNames = ReflectionTestUtils.getField(node, "tsKeyNames"); + var actualFetchMode = ReflectionTestUtils.getField(node, "fetchMode"); + var actualOrderByFetchAll = ReflectionTestUtils.getField(node, "orderByFetchAll"); + var actualAggregation = ReflectionTestUtils.getField(node, "aggregation"); + + assertThat(actualLimit).isEqualTo(5); + assertThat(actualTsKeyNames).isEqualTo(keys); + assertThat(actualFetchMode).isEqualTo("ALL"); + assertThat(actualOrderByFetchAll).isEqualTo("DESC"); + assertThat(actualAggregation).isEqualTo(Aggregation.MIN); + } + + @Test + public void givenEmptyTsKeyNames_whenOnMsg_thenTellFailure() throws TbNodeException { + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); + node.onMsg(ctxMock, msg); + + ArgumentCaptor actualException = ArgumentCaptor.forClass(Throwable.class); + then(ctxMock).should().tellFailure(eq(msg), actualException.capture()); + assertThat(actualException.getValue()) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Telemetry is not selected!"); + } + + @Test + public void givenUseMetadataIntervalPatternsIsTrueAndFetchModeIsAllAndAggregationIsMin_whenOnMsg_thenTellSuccess() throws TbNodeException { + config.setLatestTsKeyNames(List.of("temperature", "humidity")); + config.setUseMetadataIntervalPatterns(true); + config.setStartIntervalPattern("${mdStartInterval}"); + config.setEndIntervalPattern("$[msgEndInterval]"); + config.setFetchMode("ALL"); + config.setAggregation("MIN"); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + mockTimeseriesService(); + List tsKvEntries = List.of( + new BasicTsKvEntry(System.currentTimeMillis() - 5, new DoubleDataEntry("temperature", 22.4)), + new BasicTsKvEntry(System.currentTimeMillis() - 4, new DoubleDataEntry("humidity", 55.5)) + ); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(tsKvEntries)); + + long startTs = 1719220350000L; + long endTs = 1719220353000L; + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("mdStartInterval", String.valueOf(startTs)); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgEndInterval\":\"" + endTs + "\"}"); + node.onMsg(ctxMock, msg); + + List expectedReadTsKvQueryList = List.of( + new BaseReadTsKvQuery("temperature", startTs, endTs, endTs - startTs, TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE, Aggregation.MIN, "ASC"), + new BaseReadTsKvQuery("humidity", startTs, endTs, endTs - startTs, TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE, Aggregation.MIN, "ASC") + ); + verifyReadTsKvQueryList(expectedReadTsKvQueryList, false); + verifyOutgoingMsg(msg); + } + + @Test + public void givenUseMetadataIntervalPatternsIsTrueAndFetchModeIsLastAndAggregationIsMax_whenOnMsg_thenTellSuccess() throws TbNodeException { + config.setLatestTsKeyNames(List.of("temperature", "humidity")); + config.setUseMetadataIntervalPatterns(true); + config.setStartIntervalPattern("${mdStartInterval}"); + config.setEndIntervalPattern("$[msgEndInterval]"); + config.setFetchMode("LAST"); + config.setAggregation("MAX"); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + mockTimeseriesService(); + long ts = System.currentTimeMillis(); + List tsKvEntries = List.of( + new BasicTsKvEntry(ts - 5, new DoubleDataEntry("temperature", 22.4)), + new BasicTsKvEntry(ts - 4, new DoubleDataEntry("humidity", 55.5)) + ); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(tsKvEntries)); + + long startTs = 1719220350000L; + long endTs = 1719220353000L; + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("mdStartInterval", String.valueOf(startTs)); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgEndInterval\":\"" + endTs + "\"}"); + node.onMsg(ctxMock, msg); + + List expectedReadTsKvQueryList = List.of( + new BaseReadTsKvQuery("temperature", startTs, endTs, 1, 1, Aggregation.NONE, "DESC"), + new BaseReadTsKvQuery("humidity", startTs, endTs, 1, 1, Aggregation.NONE, "DESC") + ); + verifyReadTsKvQueryList(expectedReadTsKvQueryList, false); + verifyOutgoingMsg(msg); + } + + @Test + public void givenUseMetadataIntervalPatternsIsFalseAndTsKeyNamesPatternsAndFetchModeIsFirst_whenOnMsg_thenTellFailure() throws TbNodeException { + config.setLatestTsKeyNames(List.of("temperature", "${mdTsKey}", "$[msgTsKey]"));; + config.setFetchMode("FIRST"); + config.setStartInterval(6); + config.setEndInterval(1); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + mockTimeseriesService(); + String errorMsg = "Something went wrong"; + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFailedFuture(new RuntimeException(errorMsg))); + + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("mdTsKey", "humidity"); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgTsKey\":\"pressure\"}"); + node.onMsg(ctxMock, msg); + + long ts = System.currentTimeMillis(); + long startTs = ts - TimeUnit.SECONDS.toMillis(config.getStartInterval()); + long endTs = ts - TimeUnit.SECONDS.toMillis(config.getEndInterval()); + List expecetdReadTsKvQueryList = List.of( + new BaseReadTsKvQuery("temperature", startTs, endTs, 1, 1, Aggregation.NONE, "ASC"), + new BaseReadTsKvQuery("humidity", startTs, endTs, 1, 1, Aggregation.NONE, "ASC"), + new BaseReadTsKvQuery("pressure", startTs, endTs, 1, 1, Aggregation.NONE, "ASC") + ); + verifyReadTsKvQueryList(expecetdReadTsKvQueryList, true); + + ArgumentCaptor actualException = ArgumentCaptor.forClass(Throwable.class); + then(ctxMock).should().tellFailure(eq(msg), actualException.capture()); + assertThat(actualException.getValue()).isInstanceOf(RuntimeException.class).hasMessage(errorMsg); + } + + @ParameterizedTest + @MethodSource + public void givenInvalidIntervalPatterns_whenOnMsg_thenTellFailure(String startIntervalPattern, String errorMsg) throws TbNodeException { + config.setLatestTsKeyNames(List.of("${mdKey}")); + config.setUseMetadataIntervalPatterns(true); + config.setStartIntervalPattern(startIntervalPattern); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, "{\"msgStartInterval\":\"start\"}"); + node.onMsg(ctxMock, msg); + + ArgumentCaptor actualException = ArgumentCaptor.forClass(Throwable.class); + then(ctxMock).should().tellFailure(eq(msg), actualException.capture()); + assertThat(actualException.getValue()).isInstanceOf(IllegalArgumentException.class).hasMessage(errorMsg); + } + + private static Stream givenInvalidIntervalPatterns_whenOnMsg_thenTellFailure() { + return Stream.of( + Arguments.of("${mdStartInterval}", "Message value: 'mdStartInterval' is undefined"), + Arguments.of("$[msgStartInterval]", "Message value: 'msgStartInterval' has invalid format") + ); + } + + private void mockTimeseriesService() { + given(ctxMock.getTimeseriesService()).willReturn(timeseriesServiceMock); + given(ctxMock.getTenantId()).willReturn(TENANT_ID); + given(ctxMock.getDbCallbackExecutor()).willReturn(executor); + } + + private void verifyReadTsKvQueryList(List expectedReadTsKvQueryList, boolean ignoreTs) { + ArgumentCaptor> actualReadTsKvQueryCaptor = ArgumentCaptor.forClass(List.class); + then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryCaptor.capture()); + List actualReadTsKvQuery = actualReadTsKvQueryCaptor.getValue(); + for (int i = 0; i < expectedReadTsKvQueryList.size(); i++) { + assertThat(actualReadTsKvQuery.get(i)) + .usingRecursiveComparison() + .ignoringFields(!ignoreTs ? "id" : "endTs", "startTs", "id") + .isEqualTo(expectedReadTsKvQueryList.get(i)); + } + } + + private void verifyOutgoingMsg(TbMsg expectedMsg) { + ArgumentCaptor actualMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); + then(ctxMock).should().tellSuccess(actualMsgCaptor.capture()); + TbMsg actualMsg = actualMsgCaptor.getValue(); + assertThat(actualMsg).usingRecursiveComparison().ignoringFields("ctx", "metaData").isEqualTo(expectedMsg); + assertThat(actualMsg.getMetaData().getData()).containsKeys("temperature", "humidity"); + } } From b4ba613c54c2e8e8b6160d3fe7087b29288ab9a4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 4 Jul 2024 15:05:46 +0300 Subject: [PATCH 02/32] splitted to more specific tests --- .../engine/metadata/TbGetTelemetryNode.java | 58 +-- .../metadata/TbGetTelemetryNodeTest.java | 405 +++++++++++------- 2 files changed, 286 insertions(+), 177 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index 86f309ab09..4aabc606ab 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -66,19 +66,19 @@ public class TbGetTelemetryNode implements TbNode { private List tsKeyNames; private int limit; private String fetchMode; - private String orderByFetchAll; + private String orderBy; private Aggregation aggregation; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbGetTelemetryNodeConfiguration.class); tsKeyNames = config.getLatestTsKeyNames(); + if (tsKeyNames.isEmpty()) { + throw new TbNodeException("Telemetry is not selected!", true); + } limit = config.getFetchMode().equals(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL) ? validateLimit(config.getLimit()) : 1; fetchMode = config.getFetchMode(); - orderByFetchAll = config.getOrderBy(); - if (StringUtils.isEmpty(orderByFetchAll)) { - orderByFetchAll = ASC_ORDER; - } + orderBy = getOrderBy(); aggregation = parseAggregationConfig(config.getAggregation()); } @@ -91,21 +91,13 @@ public class TbGetTelemetryNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - if (tsKeyNames.isEmpty()) { - ctx.tellFailure(msg, new IllegalStateException("Telemetry is not selected!")); - } else { - try { - Interval interval = getInterval(msg); - List keys = TbNodeUtils.processPatterns(tsKeyNames, msg); - ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(interval, keys)); - DonAsynchron.withCallback(list, data -> { - var metaData = updateMetadata(data, msg, keys); - ctx.tellSuccess(TbMsg.transformMsgMetadata(msg, metaData)); - }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); - } catch (Exception e) { - ctx.tellFailure(msg, e); - } - } + Interval interval = getInterval(msg); + List keys = TbNodeUtils.processPatterns(tsKeyNames, msg); + ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(interval, keys)); + DonAsynchron.withCallback(list, data -> { + var metaData = updateMetadata(data, msg, keys); + ctx.tellSuccess(TbMsg.transformMsgMetadata(msg, metaData)); + }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } private List buildQueries(Interval interval, List keys) { @@ -115,14 +107,14 @@ public class TbGetTelemetryNode implements TbNode { interval.getEndTs() - interval.getStartTs(); return keys.stream() - .map(key -> new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), aggIntervalStep, limit, aggregation, getOrderBy())) + .map(key -> new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), aggIntervalStep, limit, aggregation, orderBy)) .collect(Collectors.toList()); } - private String getOrderBy() { + private String getOrderBy() throws TbNodeException { switch (fetchMode) { case TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL: - return orderByFetchAll; + return getOrderByFetchAll(); case TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST: return ASC_ORDER; default: @@ -130,6 +122,17 @@ public class TbGetTelemetryNode implements TbNode { } } + private String getOrderByFetchAll() throws TbNodeException { + String orderBy = config.getOrderBy(); + if (ASC_ORDER.equals(orderBy) || DESC_ORDER.equals(orderBy)) { + return orderBy; + } + if (StringUtils.isBlank(orderBy)) { + return ASC_ORDER; + } + throw new TbNodeException("Invalid fetch order selected.", true); + } + private TbMsgMetaData updateMetadata(List entries, TbMsg msg, List keys) { ObjectNode resultNode = JacksonUtil.newObjectNode(JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); if (TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL.equals(fetchMode)) { @@ -210,12 +213,11 @@ public class TbGetTelemetryNode implements TbNode { return pattern.replaceAll("[$\\[{}\\]]", ""); } - private int validateLimit(int limit) { - if (limit != 0) { - return limit; - } else { - return TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE; + private int validateLimit(int limit) throws TbNodeException { + if (limit < 2 || limit > TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE) { + throw new TbNodeException("Limit should be in a range from 2 to 1000.", true); } + return limit; } @Data diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java index 1875dd91f1..2ad575f0aa 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java @@ -22,34 +22,36 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.TestDbCallbackExecutor; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvQuery; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -77,14 +79,15 @@ public class TbGetTelemetryNodeTest { private TimeseriesService timeseriesServiceMock; @BeforeEach - public void setUp() throws Exception { + public void setUp() { node = new TbGetTelemetryNode(); config = new TbGetTelemetryNodeConfiguration().defaultConfiguration(); + config.setLatestTsKeyNames(List.of("temperature")); } @Test public void givenAggregationAsString_whenParseAggregation_thenReturnEnum() throws TbNodeException { - config.setFetchMode("ALL"); + config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); //compatibility with old configs without "aggregation" parameter @@ -107,20 +110,27 @@ public class TbGetTelemetryNodeTest { @Test public void givenAggregationWhiteSpace_whenParseAggregation_thenException() throws TbNodeException { - config.setFetchMode("ALL"); + // GIVEN + config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + // WHEN-THEN assertThatThrownBy(() -> node.parseAggregationConfig(" ")).isInstanceOf(IllegalArgumentException.class); } @Test public void givenAggregationIncorrect_whenParseAggregation_thenException() throws TbNodeException { - config.setFetchMode("ALL"); + // GIVEN + config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + // WHEN-THEN assertThatThrownBy(() -> node.parseAggregationConfig("TOP")).isInstanceOf(IllegalArgumentException.class); } @Test public void verifyDefaultConfig() { + config = new TbGetTelemetryNodeConfiguration().defaultConfiguration(); assertThat(config.getStartInterval()).isEqualTo(2); assertThat(config.getEndInterval()).isEqualTo(1); assertThat(config.getStartIntervalPattern()).isEqualTo(""); @@ -128,228 +138,325 @@ public class TbGetTelemetryNodeTest { assertThat(config.isUseMetadataIntervalPatterns()).isFalse(); assertThat(config.getStartIntervalTimeUnit()).isEqualTo(TimeUnit.MINUTES.name()); assertThat(config.getEndIntervalTimeUnit()).isEqualTo(TimeUnit.MINUTES.name()); - assertThat(config.getFetchMode()).isEqualTo("FIRST"); + assertThat(config.getFetchMode()).isEqualTo(TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST); assertThat(config.getOrderBy()).isEqualTo("ASC"); assertThat(config.getAggregation()).isEqualTo(Aggregation.NONE.name()); assertThat(config.getLimit()).isEqualTo(1000); assertThat(config.getLatestTsKeyNames()).isEmpty(); } + @Test + public void givenEmptyTsKeyNames_whenInit_thenThrowsException() { + // GIVEN + config.setLatestTsKeyNames(Collections.emptyList()); + + // WHEN-THEN + assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) + .isInstanceOf(TbNodeException.class) + .hasMessage("Telemetry is not selected!") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(true); + } + @ParameterizedTest - @MethodSource - public void givenFetchModeAndLimit_whenInit_thenVerifyLimit(String fetchMode, int limit, int expectedLimit) throws TbNodeException { - config.setFetchMode(fetchMode); + @ValueSource(ints = {-1, 0, 1, 1001, 2000}) + public void givenFetchModeAllAndLimitIsOutOfRange_whenInit_thenThrowsException(int limit) { + // GIVEN + config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); config.setLimit(limit); - node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - - var actualLimit = ReflectionTestUtils.getField(node, "limit"); - assertThat(actualLimit).isEqualTo(expectedLimit); - } - - private static Stream givenFetchModeAndLimit_whenInit_thenVerifyLimit() { - return Stream.of( - Arguments.of("FIRST", 0, 1), - Arguments.of("LAST", 10, 1), - Arguments.of("ALL", 0, 1000), - Arguments.of("ALL", 5, 5) - ); + // THEN + assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) + .isInstanceOf(TbNodeException.class) + .hasMessage("Limit should be in a range from 2 to 1000.") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(true); } @ParameterizedTest - @NullAndEmptySource - public void givenEmptyOrderBy_whenInit_thenVerify(String orderBy) throws TbNodeException { + @ValueSource(strings = {".ASC", "ascending", "DESCENDING"}) + public void givenFetchModeAllAndInvalidOrderBy_whenInit_thenThrowsException(String orderBy) { + // GIVEN + config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); + config.setLimit(2); config.setOrderBy(orderBy); - node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - - var actualOrderBy = ReflectionTestUtils.getField(node, "orderByFetchAll"); - assertThat(actualOrderBy).isEqualTo("ASC"); + // WHEN-THEN + assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) + .isInstanceOf(TbNodeException.class) + .hasMessage("Invalid fetch order selected.") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(true); } @Test - public void givenConfig_whenInit_thenVerify() throws TbNodeException { - List keys = List.of("temperature", "humidity"); - config.setLatestTsKeyNames(keys); - config.setFetchMode("ALL"); - config.setLimit(5); - config.setOrderBy("DESC"); - config.setAggregation("MIN"); + public void givenUseMetadataIntervalPatternsIsTrue_whenOnMsg_thenVerifyStartAndEndTsInQuery() throws TbNodeException { + // GIVEN + config.setUseMetadataIntervalPatterns(true); + config.setStartIntervalPattern("${mdStartInterval}"); + config.setEndIntervalPattern("$[msgEndInterval]"); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - var actualLimit = ReflectionTestUtils.getField(node, "limit"); - var actualTsKeyNames = ReflectionTestUtils.getField(node, "tsKeyNames"); - var actualFetchMode = ReflectionTestUtils.getField(node, "fetchMode"); - var actualOrderByFetchAll = ReflectionTestUtils.getField(node, "orderByFetchAll"); - var actualAggregation = ReflectionTestUtils.getField(node, "aggregation"); - - assertThat(actualLimit).isEqualTo(5); - assertThat(actualTsKeyNames).isEqualTo(keys); - assertThat(actualFetchMode).isEqualTo("ALL"); - assertThat(actualOrderByFetchAll).isEqualTo("DESC"); - assertThat(actualAggregation).isEqualTo(Aggregation.MIN); + mockTimeseriesService(); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); + + // WHEN + long startTs = 1719220350000L; + long endTs = 1719220353000L; + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("mdStartInterval", String.valueOf(startTs)); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgEndInterval\":\"" + endTs + "\"}"); + node.onMsg(ctxMock, msg); + + /// THEN + ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); + then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); + ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); + assertThat(actualReadTsKvQuery.getStartTs()).isEqualTo(startTs); + assertThat(actualReadTsKvQuery.getEndTs()).isEqualTo(endTs); } @Test - public void givenEmptyTsKeyNames_whenOnMsg_thenTellFailure() throws TbNodeException { + public void givenUseMetadataIntervalPatternsIsFalse_whenOnMsg_thenVerifyStartAndEndTsInQuery() throws TbNodeException { + // GIVEN node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + mockTimeseriesService(); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); + + // WHEN TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); node.onMsg(ctxMock, msg); - ArgumentCaptor actualException = ArgumentCaptor.forClass(Throwable.class); - then(ctxMock).should().tellFailure(eq(msg), actualException.capture()); - assertThat(actualException.getValue()) - .isInstanceOf(IllegalStateException.class) - .hasMessage("Telemetry is not selected!"); + // THEN + ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); + then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); + ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); + assertThat(actualReadTsKvQuery.getStartTs()).isLessThan(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(config.getStartInterval())); + assertThat(actualReadTsKvQuery.getEndTs()).isLessThan(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(config.getEndInterval())); } @Test - public void givenUseMetadataIntervalPatternsIsTrueAndFetchModeIsAllAndAggregationIsMin_whenOnMsg_thenTellSuccess() throws TbNodeException { - config.setLatestTsKeyNames(List.of("temperature", "humidity")); - config.setUseMetadataIntervalPatterns(true); - config.setStartIntervalPattern("${mdStartInterval}"); - config.setEndIntervalPattern("$[msgEndInterval]"); - config.setFetchMode("ALL"); - config.setAggregation("MIN"); + public void givenTsKeyNamesPatterns_whenOnMsg_thenVerifyTsKeyNamesInQuery() throws TbNodeException { + // GIVEN + config.setLatestTsKeyNames(List.of("temperature", "${mdTsKey}", "$[msgTsKey]")); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); mockTimeseriesService(); - List tsKvEntries = List.of( - new BasicTsKvEntry(System.currentTimeMillis() - 5, new DoubleDataEntry("temperature", 22.4)), - new BasicTsKvEntry(System.currentTimeMillis() - 4, new DoubleDataEntry("humidity", 55.5)) - ); - given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(tsKvEntries)); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); - long startTs = 1719220350000L; - long endTs = 1719220353000L; + // WHEN TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("mdStartInterval", String.valueOf(startTs)); - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgEndInterval\":\"" + endTs + "\"}"); + metaData.putValue("mdTsKey", "humidity"); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgTsKey\":\"pressure\"}"); node.onMsg(ctxMock, msg); - List expectedReadTsKvQueryList = List.of( - new BaseReadTsKvQuery("temperature", startTs, endTs, endTs - startTs, TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE, Aggregation.MIN, "ASC"), - new BaseReadTsKvQuery("humidity", startTs, endTs, endTs - startTs, TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE, Aggregation.MIN, "ASC") - ); - verifyReadTsKvQueryList(expectedReadTsKvQueryList, false); - verifyOutgoingMsg(msg); + // THEN + ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); + then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); + List actualKeys = actualReadTsKvQueryList.getValue().stream().map(TsKvQuery::getKey).toList(); + List expectedTsKeyNames = config.getLatestTsKeyNames().stream().map(tsKeyName -> TbNodeUtils.processPattern(tsKeyName, msg)).toList(); + assertThat(actualKeys).containsAll(expectedTsKeyNames); } - @Test - public void givenUseMetadataIntervalPatternsIsTrueAndFetchModeIsLastAndAggregationIsMax_whenOnMsg_thenTellSuccess() throws TbNodeException { - config.setLatestTsKeyNames(List.of("temperature", "humidity")); - config.setUseMetadataIntervalPatterns(true); - config.setStartIntervalPattern("${mdStartInterval}"); - config.setEndIntervalPattern("$[msgEndInterval]"); - config.setFetchMode("LAST"); - config.setAggregation("MAX"); + @ParameterizedTest + @MethodSource + public void givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery(String aggregation, Consumer verifyAggregationStepInQuery) throws TbNodeException { + // GIVEN + config.setStartInterval(5); + config.setEndInterval(1); + config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); + config.setAggregation(aggregation); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); mockTimeseriesService(); - long ts = System.currentTimeMillis(); - List tsKvEntries = List.of( - new BasicTsKvEntry(ts - 5, new DoubleDataEntry("temperature", 22.4)), - new BasicTsKvEntry(ts - 4, new DoubleDataEntry("humidity", 55.5)) - ); - given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(tsKvEntries)); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); - long startTs = 1719220350000L; - long endTs = 1719220353000L; - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("mdStartInterval", String.valueOf(startTs)); - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgEndInterval\":\"" + endTs + "\"}"); + // WHEN + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); node.onMsg(ctxMock, msg); - List expectedReadTsKvQueryList = List.of( - new BaseReadTsKvQuery("temperature", startTs, endTs, 1, 1, Aggregation.NONE, "DESC"), - new BaseReadTsKvQuery("humidity", startTs, endTs, 1, 1, Aggregation.NONE, "DESC") + // THEN + ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); + then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); + ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); + verifyAggregationStepInQuery.accept(actualReadTsKvQuery); + } + + private static Stream givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery() { + return Stream.of( + Arguments.of("", (Consumer) query -> assertThat(query.getInterval()).isEqualTo(1)), + Arguments.of("MIN", (Consumer) query -> assertThat(query.getInterval()).isEqualTo(query.getEndTs() - query.getStartTs())) ); - verifyReadTsKvQueryList(expectedReadTsKvQueryList, false); - verifyOutgoingMsg(msg); } - @Test - public void givenUseMetadataIntervalPatternsIsFalseAndTsKeyNamesPatternsAndFetchModeIsFirst_whenOnMsg_thenTellFailure() throws TbNodeException { - config.setLatestTsKeyNames(List.of("temperature", "${mdTsKey}", "$[msgTsKey]"));; - config.setFetchMode("FIRST"); - config.setStartInterval(6); - config.setEndInterval(1); + @ParameterizedTest + @MethodSource + public void givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery(String fetchMode, int limit, Consumer verifyLimitInQuery) throws TbNodeException { + // GIVEN + config.setFetchMode(fetchMode); + config.setLimit(limit); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); mockTimeseriesService(); - String errorMsg = "Something went wrong"; - given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFailedFuture(new RuntimeException(errorMsg))); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("mdTsKey", "humidity"); - TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgTsKey\":\"pressure\"}"); + // WHEN + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); node.onMsg(ctxMock, msg); - long ts = System.currentTimeMillis(); - long startTs = ts - TimeUnit.SECONDS.toMillis(config.getStartInterval()); - long endTs = ts - TimeUnit.SECONDS.toMillis(config.getEndInterval()); - List expecetdReadTsKvQueryList = List.of( - new BaseReadTsKvQuery("temperature", startTs, endTs, 1, 1, Aggregation.NONE, "ASC"), - new BaseReadTsKvQuery("humidity", startTs, endTs, 1, 1, Aggregation.NONE, "ASC"), - new BaseReadTsKvQuery("pressure", startTs, endTs, 1, 1, Aggregation.NONE, "ASC") + // THEN + ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); + then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); + ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); + verifyLimitInQuery.accept(actualReadTsKvQuery); + } + + private static Stream givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery() { + return Stream.of( + Arguments.of( + TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL, + 5, + (Consumer) query -> assertThat(query.getLimit()).isEqualTo(5)), + Arguments.of( + TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST, + TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE, + (Consumer) query -> assertThat(query.getLimit()).isEqualTo(1)) ); - verifyReadTsKvQueryList(expecetdReadTsKvQueryList, true); + } - ArgumentCaptor actualException = ArgumentCaptor.forClass(Throwable.class); - then(ctxMock).should().tellFailure(eq(msg), actualException.capture()); - assertThat(actualException.getValue()).isInstanceOf(RuntimeException.class).hasMessage(errorMsg); + @ParameterizedTest + @MethodSource + public void givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery(String fetchMode, String orderBy, Consumer verifyOrderInQuery) throws TbNodeException { + // GIVEN + config.setFetchMode(fetchMode); + config.setOrderBy(orderBy); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + mockTimeseriesService(); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); + + // WHEN + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); + node.onMsg(ctxMock, msg); + + // THEN + ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); + then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); + ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); + verifyOrderInQuery.accept(actualReadTsKvQuery); + } + + private static Stream givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery() { + return Stream.of( + Arguments.of( + TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL, + "", + (Consumer) query -> assertThat(query.getOrder()).isEqualTo("ASC")), + Arguments.of( + TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL, + "DESC", + (Consumer) query -> assertThat(query.getOrder()).isEqualTo("DESC")), + Arguments.of( + TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST, + "ASC", + (Consumer) query -> assertThat(query.getOrder()).isEqualTo("ASC")), + Arguments.of( + TbGetTelemetryNodeConfiguration.FETCH_MODE_LAST, + "ASC", + (Consumer) query -> assertThat(query.getOrder()).isEqualTo("DESC")) + ); } @ParameterizedTest @MethodSource - public void givenInvalidIntervalPatterns_whenOnMsg_thenTellFailure(String startIntervalPattern, String errorMsg) throws TbNodeException { - config.setLatestTsKeyNames(List.of("${mdKey}")); + public void givenInvalidIntervalPatterns_whenOnMsg_thenThrowsException(String startIntervalPattern, String errorMsg) throws TbNodeException { + // GIVEN config.setUseMetadataIntervalPatterns(true); config.setStartIntervalPattern(startIntervalPattern); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + // WHEN-THEN TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, "{\"msgStartInterval\":\"start\"}"); - node.onMsg(ctxMock, msg); - - ArgumentCaptor actualException = ArgumentCaptor.forClass(Throwable.class); - then(ctxMock).should().tellFailure(eq(msg), actualException.capture()); - assertThat(actualException.getValue()).isInstanceOf(IllegalArgumentException.class).hasMessage(errorMsg); + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)).isInstanceOf(IllegalArgumentException.class).hasMessage(errorMsg); } - private static Stream givenInvalidIntervalPatterns_whenOnMsg_thenTellFailure() { + private static Stream givenInvalidIntervalPatterns_whenOnMsg_thenThrowsException() { return Stream.of( Arguments.of("${mdStartInterval}", "Message value: 'mdStartInterval' is undefined"), Arguments.of("$[msgStartInterval]", "Message value: 'msgStartInterval' has invalid format") ); } + @Test + public void givenFetchModeAll_whenOnMsg_thenTellSuccessAndVerifyMsg() throws TbNodeException { + // GIVEN + config.setLatestTsKeyNames(List.of("temperature", "humidity")); + config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + mockTimeseriesService(); + long ts = System.currentTimeMillis(); + List tsKvEntries = List.of( + new BasicTsKvEntry(ts - 5, new DoubleDataEntry("temperature", 23.1)), + new BasicTsKvEntry(ts - 4, new DoubleDataEntry("temperature", 22.4)), + new BasicTsKvEntry(ts - 4, new DoubleDataEntry("humidity", 55.5)) + ); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(tsKvEntries)); + + // WHEN + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); + node.onMsg(ctxMock, msg); + + // THEN + ArgumentCaptor actualMsg = ArgumentCaptor.forClass(TbMsg.class); + then(ctxMock).should().tellSuccess(actualMsg.capture()); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("temperature", "[{\"ts\":" + (ts - 5) + ",\"value\":23.1},{\"ts\":" + (ts - 4) + ",\"value\":22.4}]"); + metaData.putValue("humidity", "[{\"ts\":" + (ts - 4) + ",\"value\":55.5}]"); + TbMsg expectedMsg = TbMsg.transformMsgMetadata(msg, metaData); + assertThat(actualMsg.getValue()).usingRecursiveComparison().ignoringFields("ctx").isEqualTo(expectedMsg); + } + + @Test + public void givenFetchModeIsFirst_whenOnMsg_thenTellSuccessAndVerifyMsg() throws TbNodeException { + // GIVEN + config.setLatestTsKeyNames(List.of("temperature", "humidity")); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + mockTimeseriesService(); + long ts = System.currentTimeMillis(); + List tsKvEntries = List.of( + new BasicTsKvEntry(ts - 4, new DoubleDataEntry("temperature", 22.4)), + new BasicTsKvEntry(ts - 4, new DoubleDataEntry("humidity", 55.5)) + ); + given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(tsKvEntries)); + + // WHEN + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); + node.onMsg(ctxMock, msg); + + // THEN + ArgumentCaptor actualMsg = ArgumentCaptor.forClass(TbMsg.class); + then(ctxMock).should().tellSuccess(actualMsg.capture()); + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("temperature", "\"22.4\""); + metaData.putValue("humidity", "\"55.5\""); + TbMsg expectedMsg = TbMsg.transformMsgMetadata(msg, metaData); + assertThat(actualMsg.getValue()).usingRecursiveComparison().ignoringFields("ctx").isEqualTo(expectedMsg); + } + private void mockTimeseriesService() { given(ctxMock.getTimeseriesService()).willReturn(timeseriesServiceMock); given(ctxMock.getTenantId()).willReturn(TENANT_ID); given(ctxMock.getDbCallbackExecutor()).willReturn(executor); } - private void verifyReadTsKvQueryList(List expectedReadTsKvQueryList, boolean ignoreTs) { - ArgumentCaptor> actualReadTsKvQueryCaptor = ArgumentCaptor.forClass(List.class); - then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryCaptor.capture()); - List actualReadTsKvQuery = actualReadTsKvQueryCaptor.getValue(); - for (int i = 0; i < expectedReadTsKvQueryList.size(); i++) { - assertThat(actualReadTsKvQuery.get(i)) - .usingRecursiveComparison() - .ignoringFields(!ignoreTs ? "id" : "endTs", "startTs", "id") - .isEqualTo(expectedReadTsKvQueryList.get(i)); - } - } - - private void verifyOutgoingMsg(TbMsg expectedMsg) { - ArgumentCaptor actualMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); - then(ctxMock).should().tellSuccess(actualMsgCaptor.capture()); - TbMsg actualMsg = actualMsgCaptor.getValue(); - assertThat(actualMsg).usingRecursiveComparison().ignoringFields("ctx", "metaData").isEqualTo(expectedMsg); - assertThat(actualMsg.getMetaData().getData()).containsKeys("temperature", "humidity"); - } } From a91caa3d097533938bae74ee62d4feb626c4c784 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 9 Aug 2024 09:10:25 +0300 Subject: [PATCH 03/32] added additional tests and moved currentTimeMillis to separate method to have control over time in tests --- .../engine/metadata/TbGetTelemetryNode.java | 6 ++++- .../metadata/TbGetTelemetryNodeTest.java | 24 ++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index 4aabc606ab..2b87e80127 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -176,7 +176,7 @@ public class TbGetTelemetryNode implements TbNode { return getIntervalFromPatterns(msg); } else { Interval interval = new Interval(); - long ts = System.currentTimeMillis(); + long ts = getCurrentTimeMillis(); interval.setStartTs(ts - TimeUnit.valueOf(config.getStartIntervalTimeUnit()).toMillis(config.getStartInterval())); interval.setEndTs(ts - TimeUnit.valueOf(config.getEndIntervalTimeUnit()).toMillis(config.getEndInterval())); return interval; @@ -220,6 +220,10 @@ public class TbGetTelemetryNode implements TbNode { return limit; } + long getCurrentTimeMillis() { + return System.currentTimeMillis(); + } + @Data @NoArgsConstructor private static class Interval { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java index 2ad575f0aa..f958e13ab5 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java @@ -32,7 +32,6 @@ import org.thingsboard.rule.engine.TestDbCallbackExecutor; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; -import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -60,7 +59,9 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.spy; import static org.mockito.BDDMockito.then; +import static org.mockito.BDDMockito.willReturn; @ExtendWith(MockitoExtension.class) public class TbGetTelemetryNodeTest { @@ -80,7 +81,7 @@ public class TbGetTelemetryNodeTest { @BeforeEach public void setUp() { - node = new TbGetTelemetryNode(); + node = spy(new TbGetTelemetryNode()); config = new TbGetTelemetryNodeConfiguration().defaultConfiguration(); config.setLatestTsKeyNames(List.of("temperature")); } @@ -222,6 +223,8 @@ public class TbGetTelemetryNodeTest { // GIVEN node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + long ts = System.currentTimeMillis(); + willReturn(ts).given(node).getCurrentTimeMillis(); mockTimeseriesService(); given(timeseriesServiceMock.findAll(any(TenantId.class), any(EntityId.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); @@ -233,8 +236,8 @@ public class TbGetTelemetryNodeTest { ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); - assertThat(actualReadTsKvQuery.getStartTs()).isLessThan(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(config.getStartInterval())); - assertThat(actualReadTsKvQuery.getEndTs()).isLessThan(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(config.getEndInterval())); + assertThat(actualReadTsKvQuery.getStartTs()).isEqualTo(ts - TimeUnit.MINUTES.toMillis(config.getStartInterval())); + assertThat(actualReadTsKvQuery.getEndTs()).isEqualTo(ts - TimeUnit.MINUTES.toMillis(config.getEndInterval())); } @Test @@ -257,8 +260,7 @@ public class TbGetTelemetryNodeTest { ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); List actualKeys = actualReadTsKvQueryList.getValue().stream().map(TsKvQuery::getKey).toList(); - List expectedTsKeyNames = config.getLatestTsKeyNames().stream().map(tsKeyName -> TbNodeUtils.processPattern(tsKeyName, msg)).toList(); - assertThat(actualKeys).containsAll(expectedTsKeyNames); + assertThat(actualKeys).containsAll(List.of("temperature", "humidity", "pressure")); } @ParameterizedTest @@ -325,6 +327,10 @@ public class TbGetTelemetryNodeTest { Arguments.of( TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST, TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE, + (Consumer) query -> assertThat(query.getLimit()).isEqualTo(1)), + Arguments.of( + TbGetTelemetryNodeConfiguration.FETCH_MODE_LAST, + 10, (Consumer) query -> assertThat(query.getLimit()).isEqualTo(1)) ); } @@ -424,9 +430,11 @@ public class TbGetTelemetryNodeTest { assertThat(actualMsg.getValue()).usingRecursiveComparison().ignoringFields("ctx").isEqualTo(expectedMsg); } - @Test - public void givenFetchModeIsFirst_whenOnMsg_thenTellSuccessAndVerifyMsg() throws TbNodeException { + @ParameterizedTest + @ValueSource(strings = {"FIRST", "LAST"}) + public void givenFetchMode_whenOnMsg_thenTellSuccessAndVerifyMsg(String fetchMode) throws TbNodeException { // GIVEN + config.setFetchMode(fetchMode); config.setLatestTsKeyNames(List.of("temperature", "humidity")); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); From 4721ed275de2d4569674b2ec6994bcb40f5b516b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 13 Aug 2024 14:08:25 +0300 Subject: [PATCH 04/32] added checks for null for fetchMode and OrderBy --- .../rule/engine/metadata/FetchMode.java | 22 +++ .../engine/metadata/TbGetTelemetryNode.java | 64 ++++--- .../TbGetTelemetryNodeConfiguration.java | 13 +- .../metadata/TbGetTelemetryNodeTest.java | 167 ++++++++++++++---- 4 files changed, 195 insertions(+), 71 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/FetchMode.java diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/FetchMode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/FetchMode.java new file mode 100644 index 0000000000..57fee39a3f --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/FetchMode.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.metadata; + +public enum FetchMode { + + FIRST, ALL, LAST + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index 2b87e80127..5faf0e40e2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -15,6 +15,7 @@ */ package org.thingsboard.rule.engine.metadata; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; @@ -35,7 +36,9 @@ import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.SortOrder.Direction; import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -50,6 +53,7 @@ import java.util.stream.Collectors; @RuleNode(type = ComponentType.ENRICHMENT, name = "originator telemetry", configClazz = TbGetTelemetryNodeConfiguration.class, + version = 1, nodeDescription = "Adds message originator telemetry for selected time range into message metadata", nodeDetails = "Useful when you need to get telemetry data set from the message originator for a specific time range " + "instead of fetching just the latest telemetry or if you need to get the closest telemetry to the fetch interval start or end. " + @@ -59,14 +63,11 @@ import java.util.stream.Collectors; configDirective = "tbEnrichmentNodeGetTelemetryFromDatabase") public class TbGetTelemetryNode implements TbNode { - private static final String DESC_ORDER = "DESC"; - private static final String ASC_ORDER = "ASC"; - private TbGetTelemetryNodeConfiguration config; private List tsKeyNames; private int limit; - private String fetchMode; - private String orderBy; + private FetchMode fetchMode; + private Direction orderBy; private Aggregation aggregation; @Override @@ -76,14 +77,17 @@ public class TbGetTelemetryNode implements TbNode { if (tsKeyNames.isEmpty()) { throw new TbNodeException("Telemetry is not selected!", true); } - limit = config.getFetchMode().equals(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL) ? validateLimit(config.getLimit()) : 1; + if (config.getFetchMode() == null) { + throw new TbNodeException("FetchMode should be specified!", true); + } + limit = FetchMode.ALL.equals(config.getFetchMode()) ? validateLimit(config.getLimit()) : 1; fetchMode = config.getFetchMode(); orderBy = getOrderBy(); aggregation = parseAggregationConfig(config.getAggregation()); } Aggregation parseAggregationConfig(String aggName) { - if (StringUtils.isEmpty(aggName) || !fetchMode.equals(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL)) { + if (StringUtils.isEmpty(aggName) || !fetchMode.equals(FetchMode.ALL)) { return Aggregation.NONE; } return Aggregation.valueOf(aggName); @@ -107,35 +111,29 @@ public class TbGetTelemetryNode implements TbNode { interval.getEndTs() - interval.getStartTs(); return keys.stream() - .map(key -> new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), aggIntervalStep, limit, aggregation, orderBy)) + .map(key -> new BaseReadTsKvQuery(key, interval.getStartTs(), interval.getEndTs(), aggIntervalStep, limit, aggregation, orderBy.name())) .collect(Collectors.toList()); } - private String getOrderBy() throws TbNodeException { + private Direction getOrderBy() throws TbNodeException { switch (fetchMode) { - case TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL: - return getOrderByFetchAll(); - case TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST: - return ASC_ORDER; + case ALL: + if (config.getOrderBy() == null) { + throw new TbNodeException("OrderBy should be specified!", true); + } + return config.getOrderBy(); + case FIRST: + return Direction.ASC; + case LAST: + return Direction.DESC; default: - return DESC_ORDER; - } - } - - private String getOrderByFetchAll() throws TbNodeException { - String orderBy = config.getOrderBy(); - if (ASC_ORDER.equals(orderBy) || DESC_ORDER.equals(orderBy)) { - return orderBy; - } - if (StringUtils.isBlank(orderBy)) { - return ASC_ORDER; + throw new TbNodeException("FetchMode '" + fetchMode + "' is not supported.", true); } - throw new TbNodeException("Invalid fetch order selected.", true); } private TbMsgMetaData updateMetadata(List entries, TbMsg msg, List keys) { ObjectNode resultNode = JacksonUtil.newObjectNode(JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); - if (TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL.equals(fetchMode)) { + if (FetchMode.ALL.equals(fetchMode)) { entries.forEach(entry -> processArray(resultNode, entry)); } else { entries.forEach(entry -> processSingle(resultNode, entry)); @@ -231,4 +229,18 @@ public class TbGetTelemetryNode implements TbNode { private Long endTs; } + @Override + public TbPair upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException { + boolean hasChanges = false; + switch (fromVersion) { + case 0 -> { + if (oldConfiguration.has("orderBy") && oldConfiguration.get("orderBy").isNull()) { + ((ObjectNode) oldConfiguration).put("orderBy", Direction.ASC.name()); + hasChanges = true; + } + } + } + return new TbPair<>(hasChanges, oldConfiguration); + } + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java index c4128a4481..3bb9ff1013 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.metadata; import lombok.Data; import org.thingsboard.rule.engine.api.NodeConfiguration; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.page.SortOrder.Direction; import java.util.Collections; import java.util.List; @@ -29,10 +30,6 @@ import java.util.concurrent.TimeUnit; @Data public class TbGetTelemetryNodeConfiguration implements NodeConfiguration { - public static final String FETCH_MODE_FIRST = "FIRST"; - public static final String FETCH_MODE_LAST = "LAST"; - public static final String FETCH_MODE_ALL = "ALL"; - public static final int MAX_FETCH_SIZE = 1000; private int startInterval; @@ -45,8 +42,8 @@ public class TbGetTelemetryNodeConfiguration implements NodeConfiguration node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) + .isInstanceOf(TbNodeException.class) + .hasMessage("FetchMode should be specified!") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(true); + } + + @Test + public void givenFetchModeAllAndOrderByIsNull_whenInit_thenThrowsException() { + // GIVEN + config.setFetchMode(FetchMode.ALL); + config.setOrderBy(null); // THEN assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) .isInstanceOf(TbNodeException.class) - .hasMessage("Limit should be in a range from 2 to 1000.") + .hasMessage("OrderBy should be specified!") .extracting(e -> ((TbNodeException) e).isUnrecoverable()) .isEqualTo(true); } @ParameterizedTest - @ValueSource(strings = {".ASC", "ascending", "DESCENDING"}) - public void givenFetchModeAllAndInvalidOrderBy_whenInit_thenThrowsException(String orderBy) { + @ValueSource(ints = {-1, 0, 1, 1001, 2000}) + public void givenFetchModeAllAndLimitIsOutOfRange_whenInit_thenThrowsException(int limit) { // GIVEN - config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); - config.setLimit(2); - config.setOrderBy(orderBy); + config.setFetchMode(FetchMode.ALL); + config.setLimit(limit); - // WHEN-THEN + // THEN assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) .isInstanceOf(TbNodeException.class) - .hasMessage("Invalid fetch order selected.") + .hasMessage("Limit should be in a range from 2 to 1000.") .extracting(e -> ((TbNodeException) e).isUnrecoverable()) .isEqualTo(true); } @@ -269,7 +283,7 @@ public class TbGetTelemetryNodeTest { // GIVEN config.setStartInterval(5); config.setEndInterval(1); - config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); + config.setFetchMode(FetchMode.ALL); config.setAggregation(aggregation); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -297,7 +311,7 @@ public class TbGetTelemetryNodeTest { @ParameterizedTest @MethodSource - public void givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery(String fetchMode, int limit, Consumer verifyLimitInQuery) throws TbNodeException { + public void givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery(FetchMode fetchMode, int limit, Consumer verifyLimitInQuery) throws TbNodeException { // GIVEN config.setFetchMode(fetchMode); config.setLimit(limit); @@ -321,15 +335,15 @@ public class TbGetTelemetryNodeTest { private static Stream givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery() { return Stream.of( Arguments.of( - TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL, + FetchMode.ALL, 5, (Consumer) query -> assertThat(query.getLimit()).isEqualTo(5)), Arguments.of( - TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST, + FetchMode.FIRST, TbGetTelemetryNodeConfiguration.MAX_FETCH_SIZE, (Consumer) query -> assertThat(query.getLimit()).isEqualTo(1)), Arguments.of( - TbGetTelemetryNodeConfiguration.FETCH_MODE_LAST, + FetchMode.LAST, 10, (Consumer) query -> assertThat(query.getLimit()).isEqualTo(1)) ); @@ -337,7 +351,7 @@ public class TbGetTelemetryNodeTest { @ParameterizedTest @MethodSource - public void givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery(String fetchMode, String orderBy, Consumer verifyOrderInQuery) throws TbNodeException { + public void givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery(FetchMode fetchMode, Direction orderBy, Consumer verifyOrderInQuery) throws TbNodeException { // GIVEN config.setFetchMode(fetchMode); config.setOrderBy(orderBy); @@ -361,20 +375,16 @@ public class TbGetTelemetryNodeTest { private static Stream givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery() { return Stream.of( Arguments.of( - TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL, - "", - (Consumer) query -> assertThat(query.getOrder()).isEqualTo("ASC")), - Arguments.of( - TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL, - "DESC", + FetchMode.ALL, + Direction.DESC, (Consumer) query -> assertThat(query.getOrder()).isEqualTo("DESC")), Arguments.of( - TbGetTelemetryNodeConfiguration.FETCH_MODE_FIRST, - "ASC", + FetchMode.FIRST, + Direction.ASC, (Consumer) query -> assertThat(query.getOrder()).isEqualTo("ASC")), Arguments.of( - TbGetTelemetryNodeConfiguration.FETCH_MODE_LAST, - "ASC", + FetchMode.LAST, + Direction.ASC, (Consumer) query -> assertThat(query.getOrder()).isEqualTo("DESC")) ); } @@ -403,7 +413,7 @@ public class TbGetTelemetryNodeTest { public void givenFetchModeAll_whenOnMsg_thenTellSuccessAndVerifyMsg() throws TbNodeException { // GIVEN config.setLatestTsKeyNames(List.of("temperature", "humidity")); - config.setFetchMode(TbGetTelemetryNodeConfiguration.FETCH_MODE_ALL); + config.setFetchMode(FetchMode.ALL); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -434,7 +444,7 @@ public class TbGetTelemetryNodeTest { @ValueSource(strings = {"FIRST", "LAST"}) public void givenFetchMode_whenOnMsg_thenTellSuccessAndVerifyMsg(String fetchMode) throws TbNodeException { // GIVEN - config.setFetchMode(fetchMode); + config.setFetchMode(FetchMode.valueOf(fetchMode)); config.setLatestTsKeyNames(List.of("temperature", "humidity")); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -467,4 +477,87 @@ public class TbGetTelemetryNodeTest { given(ctxMock.getDbCallbackExecutor()).willReturn(executor); } + private static Stream givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() { + return Stream.of( + // config for version 0 (orderBy id null) + Arguments.of(0, + """ + { + "latestTsKeyNames": [ + ], + "aggregation": "NONE", + "fetchMode": "ALL", + "orderBy": null, + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """, + true, + """ + { + "latestTsKeyNames": [ + ], + "aggregation": "NONE", + "fetchMode": "ALL", + "orderBy": "ASC", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """), + // config for version 0 (orderBy is specified) + Arguments.of(0, + """ + { + "latestTsKeyNames": [ + ], + "aggregation": "NONE", + "fetchMode": "ALL", + "orderBy": "DESC", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """, + false, + """ + { + "latestTsKeyNames": [ + ], + "aggregation": "NONE", + "fetchMode": "ALL", + "orderBy": "DESC", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """) + ); + } + + @Override + protected TbNode getTestNode() { + return node; + } } From b149957e439ec4cf0c1d02a96c2888e5233f15c9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 15 Aug 2024 16:23:26 +0300 Subject: [PATCH 05/32] added checks on init() and provided upgrade for existing nodes --- .../engine/metadata/TbGetTelemetryNode.java | 92 +++++--- .../TbGetTelemetryNodeConfiguration.java | 4 +- .../metadata/TbGetTelemetryNodeTest.java | 211 ++++++++++++------ 3 files changed, 210 insertions(+), 97 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index 5faf0e40e2..2b5190563d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -31,7 +31,6 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; @@ -75,27 +74,43 @@ public class TbGetTelemetryNode implements TbNode { this.config = TbNodeUtils.convert(configuration, TbGetTelemetryNodeConfiguration.class); tsKeyNames = config.getLatestTsKeyNames(); if (tsKeyNames.isEmpty()) { - throw new TbNodeException("Telemetry is not selected!", true); + throw new TbNodeException("Telemetry should be specified!", true); } - if (config.getFetchMode() == null) { + fetchMode = config.getFetchMode(); + if (fetchMode == null) { throw new TbNodeException("FetchMode should be specified!", true); } - limit = FetchMode.ALL.equals(config.getFetchMode()) ? validateLimit(config.getLimit()) : 1; - fetchMode = config.getFetchMode(); - orderBy = getOrderBy(); - aggregation = parseAggregationConfig(config.getAggregation()); - } - - Aggregation parseAggregationConfig(String aggName) { - if (StringUtils.isEmpty(aggName) || !fetchMode.equals(FetchMode.ALL)) { - return Aggregation.NONE; + switch (fetchMode) { + case ALL: + limit = validateLimit(config.getLimit()); + if (config.getOrderBy() == null) { + throw new TbNodeException("OrderBy should be specified!", true); + } + orderBy = config.getOrderBy(); + if (config.getAggregation() == null) { + throw new TbNodeException("Aggregation should be specified!", true); + } + aggregation = config.getAggregation(); + break; + case FIRST: + limit = 1; + orderBy = Direction.ASC; + aggregation = Aggregation.NONE; + break; + case LAST: + limit = 1; + orderBy = Direction.DESC; + aggregation = Aggregation.NONE; + break; } - return Aggregation.valueOf(aggName); } @Override public void onMsg(TbContext ctx, TbMsg msg) { Interval interval = getInterval(msg); + if (interval.getStartTs() > interval.getEndTs()) { + throw new RuntimeException("Interval start should be less than Interval end"); + } List keys = TbNodeUtils.processPatterns(tsKeyNames, msg); ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(interval, keys)); DonAsynchron.withCallback(list, data -> { @@ -115,22 +130,6 @@ public class TbGetTelemetryNode implements TbNode { .collect(Collectors.toList()); } - private Direction getOrderBy() throws TbNodeException { - switch (fetchMode) { - case ALL: - if (config.getOrderBy() == null) { - throw new TbNodeException("OrderBy should be specified!", true); - } - return config.getOrderBy(); - case FIRST: - return Direction.ASC; - case LAST: - return Direction.DESC; - default: - throw new TbNodeException("FetchMode '" + fetchMode + "' is not supported.", true); - } - } - private TbMsgMetaData updateMetadata(List entries, TbMsg msg, List keys) { ObjectNode resultNode = JacksonUtil.newObjectNode(JacksonUtil.ALLOW_UNQUOTED_FIELD_NAMES_MAPPER); if (FetchMode.ALL.equals(fetchMode)) { @@ -234,9 +233,38 @@ public class TbGetTelemetryNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0 -> { - if (oldConfiguration.has("orderBy") && oldConfiguration.get("orderBy").isNull()) { - ((ObjectNode) oldConfiguration).put("orderBy", Direction.ASC.name()); - hasChanges = true; + if (oldConfiguration.hasNonNull("fetchMode")) { + String fetchMode = oldConfiguration.get("fetchMode").asText(); + switch (fetchMode) { + case "FIRST": + ((ObjectNode) oldConfiguration).put("orderBy", Direction.ASC.name()); + ((ObjectNode) oldConfiguration).put("aggregation", Aggregation.NONE.name()); + hasChanges = true; + break; + case "LAST": + ((ObjectNode) oldConfiguration).put("orderBy", Direction.DESC.name()); + ((ObjectNode) oldConfiguration).put("aggregation", Aggregation.NONE.name()); + hasChanges = true; + break; + case "ALL": + if (oldConfiguration.has("orderBy") && + (oldConfiguration.get("orderBy").isNull() || oldConfiguration.get("orderBy").asText().isEmpty())) { + ((ObjectNode) oldConfiguration).put("orderBy", Direction.ASC.name()); + hasChanges = true; + } + if (oldConfiguration.has("aggregation") && + (oldConfiguration.get("aggregation").isNull() || oldConfiguration.get("aggregation").asText().isEmpty())) { + ((ObjectNode) oldConfiguration).put("aggregation", Aggregation.NONE.name()); + hasChanges = true; + } + break; + default: + ((ObjectNode) oldConfiguration).put("fetchMode", FetchMode.LAST.name()); + ((ObjectNode) oldConfiguration).put("orderBy", Direction.DESC.name()); + ((ObjectNode) oldConfiguration).put("aggregation", Aggregation.NONE.name()); + hasChanges = true; + break; + } } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java index 3bb9ff1013..704abeddf6 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeConfiguration.java @@ -44,7 +44,7 @@ public class TbGetTelemetryNodeConfiguration implements NodeConfiguration latestTsKeyNames; @@ -62,7 +62,7 @@ public class TbGetTelemetryNodeConfiguration implements NodeConfiguration node.parseAggregationConfig(" ")).isInstanceOf(IllegalArgumentException.class); - } - - @Test - public void givenAggregationIncorrect_whenParseAggregation_thenException() throws TbNodeException { - // GIVEN - config.setFetchMode(FetchMode.ALL); - node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); - - // WHEN-THEN - assertThatThrownBy(() -> node.parseAggregationConfig("TOP")).isInstanceOf(IllegalArgumentException.class); - } - @Test public void verifyDefaultConfig() { config = new TbGetTelemetryNodeConfiguration().defaultConfiguration(); @@ -144,7 +101,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(config.getEndIntervalTimeUnit()).isEqualTo(TimeUnit.MINUTES.name()); assertThat(config.getFetchMode()).isEqualTo(FetchMode.FIRST); assertThat(config.getOrderBy()).isEqualTo(Direction.ASC); - assertThat(config.getAggregation()).isEqualTo(Aggregation.NONE.name()); + assertThat(config.getAggregation()).isEqualTo(Aggregation.NONE); assertThat(config.getLimit()).isEqualTo(1000); assertThat(config.getLatestTsKeyNames()).isEmpty(); } @@ -157,7 +114,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { // WHEN-THEN assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) .isInstanceOf(TbNodeException.class) - .hasMessage("Telemetry is not selected!") + .hasMessage("Telemetry should be specified!") .extracting(e -> ((TbNodeException) e).isUnrecoverable()) .isEqualTo(true); } @@ -181,7 +138,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { config.setFetchMode(FetchMode.ALL); config.setOrderBy(null); - // THEN + // WHEN-THEN assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) .isInstanceOf(TbNodeException.class) .hasMessage("OrderBy should be specified!") @@ -196,7 +153,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { config.setFetchMode(FetchMode.ALL); config.setLimit(limit); - // THEN + // WHEN-THEN assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) .isInstanceOf(TbNodeException.class) .hasMessage("Limit should be in a range from 2 to 1000.") @@ -204,6 +161,33 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { .isEqualTo(true); } + @Test + public void givenFetchModeIsAllAndAggregationIsNull_whenInit_thenThrowsException() { + // GIVEN + config.setFetchMode(FetchMode.ALL); + config.setAggregation(null); + // WHEN-THEN + assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) + .isInstanceOf(TbNodeException.class) + .hasMessage("Aggregation should be specified!") + .extracting(e -> ((TbNodeException) e).isUnrecoverable()) + .isEqualTo(true); + } + + @Test + public void givenIntervalStartIsGreaterThanIntervalEnd_whenOnMsg_thenThrowsException() throws TbNodeException { + // GIVEN + config.setStartInterval(1); + config.setEndInterval(2); + + // WHEN-THEN + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); + assertThatThrownBy(() -> node.onMsg(ctxMock, msg)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Interval start should be less than Interval end"); + } + @Test public void givenUseMetadataIntervalPatternsIsTrue_whenOnMsg_thenVerifyStartAndEndTsInQuery() throws TbNodeException { // GIVEN @@ -279,7 +263,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { @ParameterizedTest @MethodSource - public void givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery(String aggregation, Consumer verifyAggregationStepInQuery) throws TbNodeException { + public void givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery(Aggregation aggregation, Consumer verifyAggregationStepInQuery) throws TbNodeException { // GIVEN config.setStartInterval(5); config.setEndInterval(1); @@ -304,8 +288,8 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { private static Stream givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery() { return Stream.of( - Arguments.of("", (Consumer) query -> assertThat(query.getInterval()).isEqualTo(1)), - Arguments.of("MIN", (Consumer) query -> assertThat(query.getInterval()).isEqualTo(query.getEndTs() - query.getStartTs())) + Arguments.of(Aggregation.NONE, (Consumer) query -> assertThat(query.getInterval()).isEqualTo(1)), + Arguments.of(Aggregation.AVG, (Consumer) query -> assertThat(query.getInterval()).isEqualTo(query.getEndTs() - query.getStartTs())) ); } @@ -479,15 +463,84 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { private static Stream givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() { return Stream.of( - // config for version 0 (orderBy id null) + // config for version 0 (fetchMode is 'FIRST' and orderBy is 'INVALID_ORDER_BY' and aggregation is 'SUM') + Arguments.of(0, + """ + { + "latestTsKeyNames": [], + "aggregation": "SUM", + "fetchMode": "FIRST", + "orderBy": "INVALID_ORDER_BY", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """, + true, + """ + { + "latestTsKeyNames": [], + "aggregation": "NONE", + "fetchMode": "FIRST", + "orderBy": "ASC", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """), + // config for version 0 (fetchMode is 'LAST' and orderBy is 'ASC' and aggregation is 'AVG') Arguments.of(0, """ { - "latestTsKeyNames": [ - ], + "latestTsKeyNames": [], + "aggregation": "AVG", + "fetchMode": "LAST", + "orderBy": "ASC", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """, + true, + """ + { + "latestTsKeyNames": [], "aggregation": "NONE", + "fetchMode": "LAST", + "orderBy": "DESC", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """), + // config for version 0 (fetchMode is 'ALL' and orderBy is empty and aggregation is null) + Arguments.of(0, + """ + { + "latestTsKeyNames": [], + "aggregation": null, "fetchMode": "ALL", - "orderBy": null, + "orderBy": "", "limit": 1000, "useMetadataIntervalPatterns": false, "startIntervalPattern": "", @@ -501,8 +554,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { true, """ { - "latestTsKeyNames": [ - ], + "latestTsKeyNames": [], "aggregation": "NONE", "fetchMode": "ALL", "orderBy": "ASC", @@ -516,13 +568,12 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { "endIntervalTimeUnit": "MINUTES" } """), - // config for version 0 (orderBy is specified) + // config for version 0 (fetchMode is 'ALL' and orderBy is 'DESC' and aggregation is 'SUM') Arguments.of(0, """ { - "latestTsKeyNames": [ - ], - "aggregation": "NONE", + "latestTsKeyNames": [], + "aggregation": "SUM", "fetchMode": "ALL", "orderBy": "DESC", "limit": 1000, @@ -538,9 +589,8 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { false, """ { - "latestTsKeyNames": [ - ], - "aggregation": "NONE", + "latestTsKeyNames": [], + "aggregation": "SUM", "fetchMode": "ALL", "orderBy": "DESC", "limit": 1000, @@ -552,6 +602,41 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { "endInterval": 1, "endIntervalTimeUnit": "MINUTES" } + """), + // config for version 0 (fetchMode is 'INVALID_MODE' and orderBy is 'INVALID_ORDER_BY' and aggregation is 'INVALID_AGGREGATION') + Arguments.of(0, + """ + { + "latestTsKeyNames": [], + "aggregation": "INVALID_AGGREGATION", + "fetchMode": "INVALID_MODE", + "orderBy": "INVALID_ORDER_BY", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } + """, + true, + """ + { + "latestTsKeyNames": [], + "aggregation": "NONE", + "fetchMode": "LAST", + "orderBy": "DESC", + "limit": 1000, + "useMetadataIntervalPatterns": false, + "startIntervalPattern": "", + "endIntervalPattern": "", + "startInterval": 2, + "startIntervalTimeUnit": "MINUTES", + "endInterval": 1, + "endIntervalTimeUnit": "MINUTES" + } """) ); } From 2bd18ee5dd2e9b5f58dc4361f6eec35bb6f0dd33 Mon Sep 17 00:00:00 2001 From: nick Date: Sun, 8 Sep 2024 21:15:20 +0300 Subject: [PATCH 06/32] lwm2m: fix bug Observe Composite --- .../rpc/AbstractRpcLwM2MIntegrationTest.java | 7 + ...cLwm2MIntegrationObserveCompositeTest.java | 144 +++++++---- .../sql/RpcLwm2mIntegrationDiscoverTest.java | 20 +- .../sql/RpcLwm2mIntegrationObserveTest.java | 236 ++++++++--------- .../store/TbInMemoryRegistrationStore.java | 160 ++---------- .../store/TbLwM2mRedisRegistrationStore.java | 135 +++------- .../uplink/DefaultLwM2mUplinkMsgHandler.java | 16 +- .../lwm2m/utils/LwM2MTransportUtil.java | 36 ++- .../server/msa/TestRestClient.java | 11 + .../lwm2m}/AbstractLwm2mClientTest.java | 241 +++++++++++++----- .../lwm2m/Lwm2mDevicesForTest.java | 20 ++ .../lwm2m/{ => client}/FwLwM2MDevice.java | 2 +- .../lwm2m/{ => client}/LwM2MTestClient.java | 66 +++-- .../client/LwM2mBinaryAppDataContainer.java | 230 +++++++++++++++++ .../lwm2m/client/LwM2mTemperatureSensor.java | 190 ++++++++++++++ .../{ => client}/LwM2mValueConverterImpl.java | 2 +- .../lwm2m/{ => client}/Lwm2mTestHelper.java | 20 +- .../lwm2m/{ => client}/SimpleLwM2MDevice.java | 38 ++- .../lwm2m/rpc/Lwm2mObserveCompositeTest.java | 52 ++++ .../lwm2m/rpc/Lwm2mObserveTest.java | 52 ++++ .../security}/Lwm2mClientNoSecTest.java | 15 +- .../security}/Lwm2mClientPskTest.java | 14 +- .../src/test/resources/lwm2m-registry/19.xml | 144 +++++++++++ .../test/resources/lwm2m-registry/3303.xml | 103 ++++++++ 24 files changed, 1390 insertions(+), 564 deletions(-) rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/{ => connectivity/lwm2m}/AbstractLwm2mClientTest.java (50%) create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/{ => client}/FwLwM2MDevice.java (98%) rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/{ => client}/LwM2MTestClient.java (79%) create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mBinaryAppDataContainer.java create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mTemperatureSensor.java rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/{ => client}/LwM2mValueConverterImpl.java (99%) rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/{ => client}/Lwm2mTestHelper.java (90%) rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/{ => client}/SimpleLwM2MDevice.java (79%) create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/{ => lwm2m/security}/Lwm2mClientNoSecTest.java (60%) rename msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/{ => lwm2m/security}/Lwm2mClientPskTest.java (64%) create mode 100644 msa/black-box-tests/src/test/resources/lwm2m-registry/19.xml create mode 100644 msa/black-box-tests/src/test/resources/lwm2m-registry/3303.xml diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java index b5f23c3e9a..1c5cf06c15 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java @@ -49,6 +49,7 @@ import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_3_9; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.TEMPERATURE_SENSOR; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.resources; +import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.fromVersionedIdToObjectId; @DaoSqlTest public abstract class AbstractRpcLwM2MIntegrationTest extends AbstractLwM2MIntegrationTest { @@ -74,7 +75,11 @@ public abstract class AbstractRpcLwM2MIntegrationTest extends AbstractLwM2MInteg protected String objectIdVer_3303; protected static AtomicInteger endpointSequence = new AtomicInteger(); protected static String DEVICE_ENDPOINT_RPC_PREF = "deviceEndpointRpc"; + + protected String idVer_3_0_0; protected String idVer_3_0_9; + protected String id_3_0_9; + protected String idVer_19_0_0; public AbstractRpcLwM2MIntegrationTest() { @@ -125,7 +130,9 @@ public abstract class AbstractRpcLwM2MIntegrationTest extends AbstractLwM2MInteg objectInstanceIdVer_5 = (String) expectedObjectIdVerInstances.stream().filter(path -> ((String) path).startsWith("/" + FIRMWARE)).findFirst().get(); objectInstanceIdVer_9 = (String) expectedObjectIdVerInstances.stream().filter(path -> ((String) path).startsWith("/" + SOFTWARE_MANAGEMENT)).findFirst().get(); + idVer_3_0_0 = objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_0; idVer_3_0_9 = objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_9; + id_3_0_9 = fromVersionedIdToObjectId(idVer_3_0_9); idVer_19_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_0; OBSERVE_ATTRIBUTES_WITH_PARAMS_RPC = diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java index 94ee1eb48e..846d4091c0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java @@ -16,14 +16,22 @@ package org.thingsboard.server.transport.lwm2m.rpc.sql; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.ResponseCode; +import org.eclipse.leshan.server.registration.Registration; import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.transport.lwm2m.rpc.AbstractRpcLwM2MIntegrationObserveTest; +import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2mUplinkMsgHandler; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_1; @@ -42,8 +50,12 @@ import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_INSTANCE_ID_0; import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.fromVersionedIdToObjectId; +@Slf4j public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MIntegrationObserveTest { + @SpyBean + DefaultLwM2mUplinkMsgHandler defaultUplinkMsgHandlerTest; + /** * ObserveComposite {"ids":["5/0/7", "5/0/5", "5/0/3", "3/0/9", "19/1/0/0"]} - Ok @@ -53,11 +65,11 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt public void testObserveCompositeAnyResources_Result_CONTENT_Value_LwM2mSingleResource_LwM2mResourceInstance() throws Exception { sendCancelObserveAllWithAwait(deviceId); String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; - String expectedIdVer5_0_5= objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; + String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + idVer_3_0_9 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); String actualValues = rpcActualResult.get("value").asText(); @@ -78,10 +90,10 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; String expectedIdVer5_0 = objectInstanceIdVer_5; String expectedIds = "[\"" + expectedIdVer19_1_0 + "\", \"" + expectedIdVer5_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actual= rpcActualResult.get("value").asText(); + String actual = rpcActualResult.get("value").asText(); assertTrue(actual.contains(fromVersionedIdToObjectId(expectedIdVer19_1_0) + "=LwM2mMultipleResource")); assertTrue(actual.contains(fromVersionedIdToObjectId(expectedIdVer5_0) + "=LwM2mObjectInstance")); } @@ -97,7 +109,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_2 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_2 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); String actualValues = rpcActualResult.get("value").asText(); @@ -117,10 +129,10 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String expectedIdVer5_0 = objectInstanceIdVer_5; String expectedIdVer5_0_2 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; String expectedIds = "[\"" + expectedIdVer5_0 + "\", \"" + expectedIdVer5_0_2 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); - String actual= rpcActualResult.get("error").asText(); + String actual = rpcActualResult.get("error").asText(); String expected = "Invalid path list : /5/0 and /5/0/2 are overlapped paths"; assertTrue(expected.equals(actual)); } @@ -133,10 +145,10 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt @Test public void testObserveCompositeThereAreObservationOneResource_Result_CONTENT_Value_ObservationAddIfAbsent() throws Exception { String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; - String expectedIdVer5_0_5= objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; + String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + idVer_3_0_9 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); String expectedResult = "/3/0/9=LwM2mSingleResource [id=9"; @@ -152,11 +164,11 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt sendCancelObserveAllWithAwait(deviceId); String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; - String expectedIdVer5_0_5= objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; + String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + idVer_3_0_9 + "\", \"" + expectedIdVer19_1_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); String actualValues = rpcActualResult.get("value").asText(); @@ -232,7 +244,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); + String actualValues = rpcActualResultReadAll.get("value").asText(); String expectedIdVer3_0_14 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_14; String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_9))); @@ -249,7 +261,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt @Test public void testObserveReadAll_Result_CONTENT_Value_SingleObservation_Only() throws Exception { sendCancelObserveAllWithAwait(deviceId); - + String expectedIdVer3_0_14 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_14; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; String actualResult3_0_9 = sendObserve("Observe", idVer_3_0_9); @@ -267,7 +279,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); + String actualValues = rpcActualResultReadAll.get("value").asText(); assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_9))); assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer3_0_14))); assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_1_0_0))); @@ -284,11 +296,11 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt sendCancelObserveAllWithAwait(deviceId); String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; - String expectedIdVer5_0_2= objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; + String expectedIdVer5_0_2 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_2 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + idVer_3_0_9 + "\", \"" + expectedIdVer19_1_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); String actualValues = rpcActualResult.get("value").asText(); @@ -298,10 +310,9 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - actualValues = rpcActualResultReadAll.get("value").asText(); + actualValues = rpcActualResultReadAll.get("value").asText(); assertFalse(actualValues.contains(fromVersionedIdToObjectId(expectedIdVer5_0_2))); - } /** @@ -312,17 +323,17 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt @Test public void testObserveCompositeAnyResources_Result_CONTENT_CancelObserveComposite_This_Result_Content_Count_5() throws Exception { sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite + // ObserveComposite String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; - String expectedIdVer5_0_5= objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; + String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + idVer_3_0_9 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // ObserveCompositeCancel - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); + // ObserveCompositeCancel + actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); assertEquals("5", rpcActualResult.get("value").asText()); @@ -338,15 +349,15 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt @Test public void testObserveCompositeOneObjectAnyResources_Result_CONTENT_CancelObserveComposite_This_Result_Content_Count_3() throws Exception { sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite + // ObserveComposite String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; String expectedIds = "[\"" + idVer_3_0_9 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // ObserveCompositeCancel - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); + // ObserveCompositeCancel + actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); assertEquals("3", rpcActualResult.get("value").asText()); @@ -363,21 +374,21 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt @Test public void testObserveCompositeAnyResources_Result_CONTENT_CancelObserveComposite_OneObjectAnyResource_Result_Content_Count_4() throws Exception { sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite + // ObserveComposite String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer3_0_9 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_9; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; - String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer3_0_9 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer3_0_9 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); awaitObserveReadAll(5, deviceId); - // ObserveCompositeCancel + // ObserveCompositeCancel expectedIds = "[\"" + objectInstanceIdVer_5 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); + actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); assertEquals("4", rpcActualResult.get("value").asText()); // CNT = 4 ("/5/0/5", "/5/0/3", "/5/0/7", "/19/1/0/0"9) @@ -385,7 +396,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); + String actualValues = rpcActualResultReadAll.get("value").asText(); assertEquals("[\"SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer3_0_9) + "\"]", actualValues); } @@ -397,26 +408,26 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt @Test public void testObserveOneObjectAnyResources_Result_CONTENT_Cancel_OneResourceFromObjectAnyResource_Result_BAD_REQUEST_Cancel_OneObject_Result_CONTENT() throws Exception { sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite + // ObserveComposite sendCancelObserveAllWithAwait(deviceId); String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; - String expectedIds = "[\"" + objectIdVer_3 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String expectedIds = "[\"" + objectIdVer_3 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // ObserveCompositeCancel + // ObserveCompositeCancel expectedIds = "[\"" + expectedIdVer19_1_0_0 + "\", \"" + idVer_3_0_9 + "\"]"; - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); + actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); String expectedValue = "for observation path [" + fromVersionedIdToObjectId(objectIdVer_3) + "], that includes this observation path [" + fromVersionedIdToObjectId(idVer_3_0_9); assertTrue(rpcActualResult.get("error").asText().contains(expectedValue)); - // ObserveCompositeCancel + // ObserveCompositeCancel expectedIds = "[\"" + objectIdVer_3 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); + actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); assertEquals("2", rpcActualResult.get("value").asText()); @@ -424,12 +435,12 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); + String actualValues = rpcActualResultReadAll.get("value").asText(); assertEquals("[\"SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer5_0_3) + "\"]", actualValues); } - /** + /** * ObserveComposite {"ids":["/3/0/9", "/3/0/14", "/5/0/3", "/3/0/15", "/19/1/0/0"]} - Ok * ObserveCancel {"id":"/3/0/9"} -> INTERNAL_SERVER_ERROR * ObserveCompositeCancel {"ids":["/3/0/9", "/19/1/0/0", "/3]} - Ok @@ -439,19 +450,19 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt @Test public void testObserveCompositeAnyResources_Result_CONTENT_CancelObserveComposite_OneResource_OneObjectAnyResource_Result_Content_Count_4() throws Exception { sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite + // ObserveComposite sendCancelObserveAllWithAwait(deviceId); String expectedIdVer3_0_14 = objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_14; - String expectedIdVer3_0_15= objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_15; + String expectedIdVer3_0_15 = objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_15; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; - String expectedIds = "[\"" + idVer_3_0_9 + "\", \"" + expectedIdVer3_0_14 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer3_0_15 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + String expectedIds = "[\"" + idVer_3_0_9 + "\", \"" + expectedIdVer3_0_14 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer3_0_15 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // ObserveCompositeCancel + // ObserveCompositeCancel expectedIds = "[\"" + idVer_3_0_9 + "\", \"" + expectedIdVer19_1_0_0 + "\", \"" + objectIdVer_3 + "\"]"; - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); + actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); assertEquals("4", rpcActualResult.get("value").asText()); @@ -459,17 +470,44 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); + String actualValues = rpcActualResultReadAll.get("value").asText(); assertEquals("[\"SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer5_0_3) + "\"]", actualValues); } + /** + * ObserveCancelAll + * Observe {"id":"/3/0/9"} + * updateRegistration + * idResources_/3/0/9 => updateAttrTelemetry >= 10 times + */ + @Test + public void testObserveCompositeResource_Update_AfterUpdateRegistration() throws Exception { + sendCancelObserveAllWithAwait(deviceId); + + int cntUpdate = 3; + verify(defaultUplinkMsgHandlerTest, timeout(50000).atLeast(cntUpdate)) + .updatedReg(Mockito.any(Registration.class)); + + log.warn("After ObserveReadAll after cancel - send composite observe /3303_1.0/0/5700"); + + String expectedKey3_0_9 = RESOURCE_ID_NAME_3_9; + String expectedKey3_0_14 = RESOURCE_ID_NAME_3_14; + String expectedKey19_0_0 = RESOURCE_ID_NAME_19_0_0; + String expectedKey19_1_0 = RESOURCE_ID_NAME_19_1_0; + String expectedKeys = "[\"" + expectedKey3_0_9 + "\", \"" + expectedKey3_0_14 + "\", \"" + expectedKey19_0_0 + "\", \"" + expectedKey19_1_0 + "\", \"" + expectedKey3_0_9 + "\"]"; + String actualResult = sendCompositeRPCByKeys("ObserveComposite", expectedKeys); + + + cntUpdate = 20; + verify(defaultUplinkMsgHandlerTest, timeout(50000).atLeast(cntUpdate)) + .updateAttrTelemetry(Mockito.any(Registration.class), argThat(arg -> arg.equals(idVer_3_0_9) || arg.equals(idVer_19_0_0))); + } private String sendObserve(String method, String params) throws Exception { String sendRpcRequest; if (params == null) { sendRpcRequest = "{\"method\": \"" + method + "\"}"; - } - else { + } else { sendRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"id\": \"" + params + "\"}}"; } return doPostAsync("/api/plugins/rpc/twoway/" + deviceId, sendRpcRequest, String.class, status().isOk()); @@ -481,7 +519,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt } private String sendCompositeRPCByKeys(String method, String keys) throws Exception { - String setRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"keys\":" + keys + "}}"; - return doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setRpcRequest, String.class, status().isOk()); + String sendRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"keys\":" + keys + "}}"; + return doPostAsync("/api/plugins/rpc/twoway/" + deviceId, sendRpcRequest, String.class, status().isOk()); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationDiscoverTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationDiscoverTest.java index e1717c9b61..9518010f6a 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationDiscoverTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationDiscoverTest.java @@ -23,6 +23,8 @@ import org.eclipse.leshan.core.link.LinkParseException; import org.eclipse.leshan.core.node.LwM2mPath; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.transport.lwm2m.config.TbLwM2mVersion; import org.thingsboard.server.transport.lwm2m.rpc.AbstractRpcLwM2MIntegrationTest; import java.util.Arrays; @@ -33,9 +35,10 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_KEY; +import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_2; -import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.convertObjectIdToVerId; public class RpcLwm2mIntegrationDiscoverTest extends AbstractRpcLwM2MIntegrationTest { @@ -172,4 +175,19 @@ public class RpcLwm2mIntegrationDiscoverTest extends AbstractRpcLwM2MIntegration String setRpcRequest = "{\"method\": \"Discover\", \"params\": {\"id\": \"" + path + "\"}}"; return doPostAsync("/api/plugins/rpc/twoway/" + deviceId, setRpcRequest, String.class, status().isOk()); } + + private String convertObjectIdToVerId(String path, String ver) { + ver = ver != null ? ver : TbLwM2mVersion.VERSION_1_0.getVersion().toString(); + try { + String[] keyArray = path.split(LWM2M_SEPARATOR_PATH); + if (keyArray.length > 1) { + keyArray[1] = keyArray[1] + LWM2M_SEPARATOR_KEY + ver; + return StringUtils.join(keyArray, LWM2M_SEPARATOR_PATH); + } else { + return path; + } + } catch (Exception e) { + return null; + } + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java index 5a849e70e4..8fdf1994b6 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java @@ -52,10 +52,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserveReadAll_Count_2_CancelAll_Count_0_Ok() throws Exception { - String actualResultReadAll = sendRpcObserve("ObserveReadAll", null); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actualValuesReadAll = rpcActualResult.get("value").asText(); + String actualValuesReadAll = sendRpcObserveWithResultValue("ObserveReadAll", null); assertEquals(2, actualValuesReadAll.split(",").length); String expected = "\"SingleObservation:/19/0/0\""; assertTrue(actualValuesReadAll.contains(expected)); @@ -70,15 +67,10 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserveOneResource_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception { sendCancelObserveAllWithAwait(deviceId); - String idVer_3_0_9 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_9; - String actualResult = sendRpcObserve("Observe", idVer_3_0_9); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertTrue(rpcActualResult.get("value").asText().contains("LwM2mSingleResource")); - assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceId)).get()); + sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0_9); int cntUpdate = 3; - verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate)) + verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate)) .onUpdateValueAfterReadResponse(Mockito.any(Registration.class), eq(idVer_3_0_9), Mockito.any(ReadResponse.class)); } @@ -90,13 +82,10 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO public void testObserveOneObjectInstance_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception { sendCancelObserveAllWithAwait(deviceId); String idVer_3_0 = objectInstanceIdVer_3; - String actualResult = sendRpcObserve("Observe", idVer_3_0); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertTrue(rpcActualResult.get("value").asText().contains("LwM2mSingleResource")); - assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceId)).get()); + sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0); + int cntUpdate = 3; - verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate)) + verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate)) .updateAttrTelemetry(Mockito.any(Registration.class), eq(idVer_3_0_9)); } @@ -108,17 +97,13 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO public void testObserveOneObject_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception { sendCancelObserveAllWithAwait(deviceId); String idVer_3_0 = objectInstanceIdVer_3; - String actualResult = sendRpcObserve("Observe", idVer_3_0); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertTrue(rpcActualResult.get("value").asText().contains("LwM2mSingleResource")); - assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceId)).get()); + sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0); + int cntUpdate = 3; - verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate)) + verify(defaultUplinkMsgHandlerTest, timeout(10000).times(cntUpdate)) .updateAttrTelemetry(Mockito.any(Registration.class), eq(idVer_3_0_9)); } - /** * Repeated request on Observe * Observe {"id":"/3_1.2/0/0"} @@ -126,15 +111,10 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveRepeated_Result_CONTENT_AddIfAbsent() throws Exception { - String idVer_3_0_0 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_0; - String actualResult = sendRpcObserve("Observe", idVer_3_0_0); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - actualResult = sendRpcObserve("Observe", idVer_3_0_0); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + sendRpcObserveWithResultValue("Observe", idVer_3_0_0); + String rpcActualResult = sendRpcObserveWithResultValue("Observe", idVer_3_0_0); String expected = "LwM2mSingleResource [id=0"; - assertTrue(rpcActualResult.get("value").asText().contains(expected)); + assertTrue(rpcActualResult.contains(expected)); } /** @@ -143,15 +123,14 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveWithBadVersion_Result_BadRequest_ErrorMsg_BadVersionMustBe_Ver() throws Exception { - String expectedInstance = (String) expectedInstances.stream().filter(path -> !((String)path).contains("_")).findFirst().get(); + String expectedInstance = (String) expectedInstances.stream().filter(path -> !((String) path).contains("_")).findFirst().get(); LwM2mPath expectedPath = new LwM2mPath(expectedInstance); int expectedResource = lwM2MTestClient.getLeshanClient().getObjectTree().getObjectEnablers().get(expectedPath.getObjectId()).getObjectModel().resources.entrySet().stream().findAny().get().getKey(); String ver = lwM2MTestClient.getLeshanClient().getObjectTree().getObjectEnablers().get(expectedPath.getObjectId()).getObjectModel().version; String expectedId = "/" + expectedPath.getObjectId() + "_" + Version.MAX + "/" + expectedPath.getObjectInstanceId() + "/" + expectedResource; - String actualResult = sendRpcObserve("Observe", expectedId); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + ObjectNode rpcActualResult = sendRpcObserveWithResult("Observe", expectedId); assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); - String expected = "Specified resource id " + expectedId +" is not valid version! Must be version: " + ver; + String expected = "Specified resource id " + expectedId + " is not valid version! Must be version: " + ver; assertEquals(expected, rpcActualResult.get("error").asText()); } @@ -162,10 +141,9 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveNoImplementedInstanceOnDevice_Result_NotFound() throws Exception { - String objectInstanceIdVer = (String) expectedObjectIdVers.stream().filter(path -> ((String)path).contains("/" + ACCESS_CONTROL)).findFirst().get(); + String objectInstanceIdVer = (String) expectedObjectIdVers.stream().filter(path -> ((String) path).contains("/" + ACCESS_CONTROL)).findFirst().get(); String expected = objectInstanceIdVer + "/" + OBJECT_INSTANCE_ID_0; - String actualResult = sendRpcObserve("Observe", expected); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + ObjectNode rpcActualResult = sendRpcObserveWithResult("Observe", expected); assertEquals(ResponseCode.NOT_FOUND.getName(), rpcActualResult.get("result").asText()); } @@ -177,8 +155,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserveNoImplementedResourceOnDeviceValueNull_Result_BadRequest() throws Exception { String expected = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_3; - String actualResult = sendRpcObserve("Observe", expected); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + ObjectNode rpcActualResult = sendRpcObserveWithResult("Observe", expected); String expectedValue = "value MUST NOT be null"; assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); assertEquals(expectedValue, rpcActualResult.get("error").asText()); @@ -191,9 +168,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserveResourceNotRead_Result_METHOD_NOT_ALLOWED() throws Exception { String expectedId = objectInstanceIdVer_5 + "/" + RESOURCE_ID_0; - sendRpcObserve("Observe", expectedId); - String actualResult = sendRpcObserve("Observe", expectedId); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + ObjectNode rpcActualResult = sendRpcObserveWithResult("Observe", expectedId); assertEquals(ResponseCode.METHOD_NOT_ALLOWED.getName(), rpcActualResult.get("result").asText()); } @@ -204,9 +179,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserveExecuteResource_Result_METHOD_NOT_ALLOWED() throws Exception { String expectedId = objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; - sendRpcObserve("Observe", expectedId); - String actual = sendRpcObserve("Observe", expectedId); - ObjectNode rpcActual = JacksonUtil.fromString(actual, ObjectNode.class); + ObjectNode rpcActual = sendRpcObserveWithResult("Observe", expectedId); assertEquals(ResponseCode.METHOD_NOT_ALLOWED.getName(), rpcActual.get("result").asText()); } @@ -217,15 +190,10 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveRepeatedRequestObserveOnDevice_Result_CONTENT_PutIfAbsent() throws Exception { - String idVer_3_0_0 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_0; - String actualResult = sendRpcObserve("Observe", idVer_3_0_0); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - actualResult = sendRpcObserve("Observe", idVer_3_0_0); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + sendRpcObserveWithResultValue("Observe", idVer_3_0_0); + String rpcActualResult = sendRpcObserveWithResultValue("Observe", idVer_3_0_0); String expected = "LwM2mSingleResource [id=0"; - assertTrue(rpcActualResult.get("value").asText().contains(expected)); + assertTrue(rpcActualResult.contains(expected)); } /** @@ -236,20 +204,12 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserve_Result_CONTENT_ONE_PATH_PreviousObservation_CONTAINCE_OTHER_CurrentObservation() throws Exception { sendCancelObserveAllWithAwait(deviceId); - // "3/0/9" - String idVer_3_0_9 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_9; - String actualResult3_0_9 = sendRpcObserve("Observe", idVer_3_0_9); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult3_0_9, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // "3" - String actualResult3 = sendRpcObserve("Observe", objectIdVer_3); - rpcActualResult = JacksonUtil.fromString(actualResult3, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // PreviousObservation "3/0/9" change to CurrentObservation "3" - String actualResultReadAll = sendRpcObserve("ObserveReadAll", null); - rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actualValuesReadAll = rpcActualResult.get("value").asText(); + // "3/0/9" + sendRpcObserveWithResultValue("Observe", idVer_3_0_9); + // "3" + sendRpcObserveWithResultValue("Observe", objectIdVer_3); + // PreviousObservation "3/0/9" change to CurrentObservation "3" + String actualValuesReadAll = sendRpcObserveWithResultValue("ObserveReadAll", null); assertEquals(1, actualValuesReadAll.split(",").length); String expected = "\"SingleObservation:/3\""; assertTrue(actualValuesReadAll.contains(expected)); @@ -263,28 +223,17 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserve_Result_CONTENT_ONE_PATH_CurrentObservation_CONTAINCE_OTHER_PreviousObservation() throws Exception { sendCancelObserveAllWithAwait(deviceId); + // "3" + sendRpcObserveWithResultValue("Observe", objectIdVer_3); + // "3/0/0"; WARN: - Token collision ? existing observation [/3] includes input observation [/3/0/0] + sendRpcObserveWithResultValue("Observe", idVer_3_0_0); - // "3" - String actualResult3 = sendRpcObserve("Observe", objectIdVer_3); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult3, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - - // "3/0/0"; WARN: - Token collision ? existing observation [/3] includes input observation [/3/0/0] - String idVer_3_0_0 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_0; - String actualResult3_0_0 = sendRpcObserve("Observe", idVer_3_0_0); - rpcActualResult = JacksonUtil.fromString(actualResult3_0_0, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - - String actualResultReadAll = sendRpcObserve("ObserveReadAll", null); - rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actualValuesReadAll = rpcActualResult.get("value").asText(); + String actualValuesReadAll = sendRpcObserveWithResultValue("ObserveReadAll", null); assertEquals(1, actualValuesReadAll.split(",").length); String expected = "\"SingleObservation:/3\""; assertTrue(actualValuesReadAll.contains(expected)); } - /** * Observe {"id":"/3/0/9"} * ObserveCancel {"id":"/3/0/9"} @@ -293,24 +242,15 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO public void testObserveResource_ObserveCancelResource_Result_CONTENT_Count_1() throws Exception { sendCancelObserveAllWithAwait(deviceId); - String expectedId_3_0_9 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_9; - sendRpcObserve("Observe", expectedId_3_0_9); - String actualResultReadAll = sendRpcObserve("ObserveReadAll", null); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actualValuesReadAll = rpcActualResult.get("value").asText(); + String actualValuesReadAll = sendRpcObserveReadAllWithResult(idVer_3_0_9); assertEquals(1, actualValuesReadAll.split(",").length); - String expected = "\"SingleObservation:" + fromVersionedIdToObjectId(expectedId_3_0_9) + "\""; + String expected = "\"SingleObservation:" + id_3_0_9 + "\""; assertTrue(actualValuesReadAll.contains(expected)); // cancel observe "/3_1.2/0/9" - String actualResult = sendRpcObserve("ObserveCancel", expectedId_3_0_9); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("1", rpcActualResult.get("value").asText()); + sendRpcObserveWithResultValue("ObserveCancel", idVer_3_0_9); } - /** * Observe {"id":"/3"} * ObserveCancel {"id":"/3/0/9"} -> INTERNAL_SERVER_ERROR @@ -320,29 +260,19 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO public void testObserveObject_ObserveCancelOneResource_Result_INTERNAL_SERVER_ERROR_Than_Cancel_ObserveObject_Result_CONTENT_Count_1() throws Exception { sendCancelObserveAllWithAwait(deviceId); - String expectedId_3 = objectIdVer_3; - sendRpcObserve("Observe", expectedId_3); - String actualResultReadAll = sendRpcObserve("ObserveReadAll", null); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actualValuesReadAll = rpcActualResult.get("value").asText(); + String actualValuesReadAll = sendRpcObserveReadAllWithResult(objectIdVer_3); assertEquals(1, actualValuesReadAll.split(",").length); - String expected = "\"SingleObservation:" + fromVersionedIdToObjectId(expectedId_3) + "\""; + String expected = "\"SingleObservation:" + fromVersionedIdToObjectId(objectIdVer_3) + "\""; assertTrue(actualValuesReadAll.contains(expected)); - // cancel observe "/3_1.2/0/9" - String expectedId_3_0_9 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_9; - String actualResult = sendRpcObserve("ObserveCancel", expectedId_3_0_9); - String expectedValue = "for observation path [" + fromVersionedIdToObjectId(objectIdVer_3) + "], that includes this observation path [" + fromVersionedIdToObjectId(expectedId_3_0_9); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + // cancel observe "/3_1.2/0/9" + ObjectNode rpcActualResult = sendRpcObserveWithResult("ObserveCancel", idVer_3_0_9); + String expectedValue = "for observation path [" + fromVersionedIdToObjectId(objectIdVer_3) + "], that includes this observation path [" + id_3_0_9; assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); assertTrue(rpcActualResult.get("error").asText().contains(expectedValue)); - // cancel observe "/3_1.2" - actualResult = sendRpcObserve("ObserveCancel", expectedId_3); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("1", rpcActualResult.get("value").asText()); + // cancel observe "/3_1.2" + sendRpcObserveWithResultValue("ObserveCancel", objectIdVer_3); } /** @@ -353,29 +283,73 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO @Test public void testObserveResource_ObserveCancelObject_Result_CONTENT_Count_1() throws Exception { sendCancelObserveAllWithAwait(deviceId); - String expectedId_3_0_0 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_0; - sendRpcObserve("Observe", expectedId_3_0_0); - String expectedId_3_0_9 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_9; - sendRpcObserve("Observe", expectedId_3_0_9); - String actualResultReadAll = sendRpcObserve("ObserveReadAll", null); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + sendRpcObserveWithWithTwoResource(idVer_3_0_0, idVer_3_0_9); + String rpcActualResul = sendRpcObserveWithResultValue("ObserveReadAll", null); + assertEquals(2, rpcActualResul.split(",").length); + String expected_3_0_0 = "\"SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_0) + "\""; + String expected_3_0_9 = "\"SingleObservation:" + id_3_0_9 + "\""; + assertTrue(rpcActualResul.contains(expected_3_0_0)); + assertTrue(rpcActualResul.contains(expected_3_0_9)); + + // cancel observe "/3_1.2" + String expectedId_3 = objectIdVer_3; + String rpcActualResult = sendRpcObserveWithResultValue("ObserveCancel", expectedId_3); + assertEquals("2", rpcActualResult); + } + + /** + * ObserveCancelAll + * Observe {"id":"3_1.2/0/9"} + * updateRegistration + * idResources_3_1.2/0/9 => updateAttrTelemetry >= 10 times + */ + @Test + public void testObserveResource_Update_AfterUpdateRegistration() throws Exception { + sendCancelObserveAllWithAwait(deviceId); + + int cntUpdate = 3; + verify(defaultUplinkMsgHandlerTest, timeout(50000).atLeast(cntUpdate)) + .updatedReg(Mockito.any(Registration.class)); + + sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0_9); + + cntUpdate = 10; + verify(defaultUplinkMsgHandlerTest, timeout(50000).atLeast(cntUpdate)) + .updateAttrTelemetry(Mockito.any(Registration.class), eq(idVer_3_0_9)); + } + + private void sendRpcObserveWithWithTwoResource(String expectedId_1, String expectedId_2) throws Exception { + sendRpcObserve("Observe", expectedId_1); + sendRpcObserve("Observe", expectedId_2); + } + + private String sendRpcObserveReadAllWithResult(String params) throws Exception { + sendRpcObserve("Observe", params); + ObjectNode rpcActualResult = sendRpcObserveWithResult("ObserveReadAll", null); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actualValuesReadAll = rpcActualResult.get("value").asText(); - assertEquals(2, actualValuesReadAll.split(",").length); - String expected_3_0_0 = "\"SingleObservation:" + fromVersionedIdToObjectId(expectedId_3_0_0) + "\""; - String expected_3_0_9 = "\"SingleObservation:" + fromVersionedIdToObjectId(expectedId_3_0_9) + "\""; - assertTrue(actualValuesReadAll.contains(expected_3_0_0)); - assertTrue(actualValuesReadAll.contains(expected_3_0_9)); + return rpcActualResult.get("value").asText(); + } - // cancel observe "/3_1.2" - String expectedId_3 = objectIdVer_3; - String actualResult = sendRpcObserve("ObserveCancel", expectedId_3); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + private ObjectNode sendRpcObserveWithResult(String method, String params) throws Exception { + String actualResultReadAll = sendRpcObserve(method, params); + return JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + } + + private String sendRpcObserveWithResultValue(String method, String params) throws Exception { + String actualResultReadAll = sendRpcObserve(method, params); + ObjectNode rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("2", rpcActualResult.get("value").asText()); + return rpcActualResult.get("value").asText(); + } + + private void sendRpcObserveWithContainsLwM2mSingleResource(String params) throws Exception { + String rpcActualResult = sendRpcObserveWithResultValue("Observe", params); + assertTrue(rpcActualResult.contains("LwM2mSingleResource")); + assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceId)).get()); } private String sendRpcObserve(String method, String params) throws Exception { return sendObserve(method, params, deviceId); } } + diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java index 138d0d6af8..8fae6a478e 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java @@ -15,25 +15,16 @@ */ package org.thingsboard.server.transport.lwm2m.server.store; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.coap.Token; -import org.eclipse.californium.core.network.RandomTokenGenerator; import org.eclipse.californium.core.network.TokenGenerator; -import org.eclipse.californium.core.network.TokenGenerator.Scope; import org.eclipse.leshan.core.Destroyable; import org.eclipse.leshan.core.Startable; import org.eclipse.leshan.core.Stoppable; -import org.eclipse.leshan.core.model.ObjectModel; -import org.eclipse.leshan.core.model.ResourceModel; -import org.eclipse.leshan.core.node.LwM2mPath; import org.eclipse.leshan.core.observation.CompositeObservation; import org.eclipse.leshan.core.observation.Observation; import org.eclipse.leshan.core.observation.ObservationIdentifier; import org.eclipse.leshan.core.observation.SingleObservation; import org.eclipse.leshan.core.peer.LwM2mIdentity; -import org.eclipse.leshan.core.request.ContentFormat; import org.eclipse.leshan.core.util.NamedThreadFactory; import org.eclipse.leshan.server.registration.Deregistration; import org.eclipse.leshan.server.registration.ExpirationListener; @@ -41,14 +32,12 @@ import org.eclipse.leshan.server.registration.Registration; import org.eclipse.leshan.server.registration.RegistrationStore; import org.eclipse.leshan.server.registration.RegistrationUpdate; import org.eclipse.leshan.server.registration.UpdatedRegistration; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.server.LwM2mVersionedModelProvider; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -60,13 +49,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import static org.eclipse.leshan.core.californium.ObserveUtil.CTX_CF_OBERSATION; -import static org.eclipse.leshan.core.californium.ObserveUtil.extractSerializedObservation; - @Slf4j public class TbInMemoryRegistrationStore implements RegistrationStore, Startable, Stoppable, Destroyable { @@ -108,7 +93,7 @@ public class TbInMemoryRegistrationStore implements RegistrationStore, Startable this.schedExecutor = schedExecutor; this.cleanPeriod = cleanPeriodInSec; this.modelProvider = modelProvider; - this.config = config; + this.config = config; } /* *************** Leshan Registration API **************** */ @@ -255,138 +240,60 @@ public class TbInMemoryRegistrationStore implements RegistrationStore, Startable List removed = new ArrayList<>(); try { lock.writeLock().lock(); - if (!regsByRegId.containsKey(registrationId)) { throw new IllegalStateException(String.format( "can not add observation %s there is no registration with id %s", observation, registrationId)); } - - if (observation instanceof SingleObservation) { - if (validateObserveResource(((SingleObservation)observation).getPath(), registrationId)) { - updateSingleObservation(registrationId, (SingleObservation) observation, addIfAbsent, removed); - // cancel existing observations for the same path and registration id. - cancelObservation (observation, registrationId, removed); - } - } else { - ContentFormat ct = ((CompositeObservation) observation).getResponseContentFormat(); - Map ctx = observation.getContext(); - String serializedObservation = extractSerializedObservation(observation); - JsonNode nodeSerObs = JacksonUtil.toJsonNode(serializedObservation); - ((CompositeObservation)observation).getPaths().forEach(path -> { - if (validateObserveResource(path, registrationId)) { - String serializedObs = createSerializedSingleObservation(nodeSerObs, path.toString()); - Observation singleObservation = createSingleObservation(registrationId, path, ct, ctx, serializedObs, getTokenGenerator()); - updateSingleObservation(registrationId, (SingleObservation) singleObservation, addIfAbsent, removed); - // cancel existing observations for the same path and registration id. - cancelObservation (singleObservation, registrationId, removed); - } - }); - } - + updateObservation(registrationId, observation, addIfAbsent, removed); } finally { lock.writeLock().unlock(); } - return removed; } - private boolean validateObserveResource(LwM2mPath path, String registrationId){ - // check if the resource is readable. - if (path.isResource() || path.isResourceInstance()) { - ObjectModel objectModel = modelProvider.getObjectModel(getRegistration(registrationId)).getObjectModel(path.getObjectId()); - ResourceModel resourceModel = objectModel == null ? null : objectModel.resources.get(path.getResourceId()); - if (resourceModel == null) { - return false; - } else if (!resourceModel.operations.isReadable()) { - return false; - } else if (path.isResourceInstance() && !resourceModel.multiple) { - return false; - } - } - return true; - } - - private void updateSingleObservation (String registrationId, SingleObservation observation, boolean addIfAbsent, List removed) { - // Absorption by existing Observations - + private void updateObservation(String registrationId, Observation observation, boolean addIfAbsent, List removed) { + // Absorption by existing Observations Observation previousObservation = null; - SingleObservation existingObservation = null; - ObservationIdentifier id = observation.getId(); if (addIfAbsent) { if (!obsByToken.containsKey(id)) { - existingObservation = validateByAbsorptionExistingObservations(observation); - if (existingObservation == null) { - obsByToken.put(id, observation); - } else if (!existingObservation.getPath().equals(observation.getPath())){ - obsByToken.put(id, observation); - previousObservation = obsByToken.get(existingObservation.getId()); - } + previousObservation = obsByToken.put(id, observation); + } else { + obsByToken.put(id, observation); } } else { previousObservation = obsByToken.put(id, observation); } + if (!tokensByRegId.containsKey(registrationId)) { tokensByRegId.put(registrationId, new HashSet()); } - - if (existingObservation == null || !existingObservation.getPath().equals(observation.getPath())) { - tokensByRegId.get(registrationId).add(id); - } + tokensByRegId.get(registrationId).add(id); // log any collisions - if (addIfAbsent && previousObservation != null) { - if (!existingObservation.getPath().equals(observation.getPath())) { - removed.add(previousObservation); - log.warn("Token collision ? observation [{}] will be replaced by observation [{}], that this observation includes input observation [{}]!", - previousObservation, observation, observation); - } else { - log.warn("Token collision ? existing observation [{}] includes input observation [{}]", - existingObservation, observation); - } + if (previousObservation != null) { + removed.add(previousObservation); + log.warn("Token collision ? observation [{}] will be replaced by observation [{}] ", + previousObservation, observation); } - } - - private SingleObservation validateByAbsorptionExistingObservations (SingleObservation observation) { - LwM2mPath pathObservation = observation.getPath(); - AtomicReference result = new AtomicReference<>(); - obsByToken.values().stream().forEach(obs -> { - LwM2mPath pathObs = ((SingleObservation)obs).getPath(); - if ((!pathObservation.equals(pathObs) && pathObs.startWith(pathObservation)) || // pathObs = "3/0/9"-> pathObservation = "3" - (pathObservation.equals(pathObs) && !observation.getId().equals(obs.getId()))) { - result.set((SingleObservation)obs); - } else if (!pathObservation.equals(pathObs) && pathObservation.startWith(pathObs)) { // pathObs = "3" -> pathObservation = "3/0/9" - result.set(observation); + // cancel existing observations for the same path and registration id. + for (Observation obs : unsafeGetObservations(registrationId)) { + if (areTheSamePaths(observation, obs) && !observation.getId().equals(obs.getId())) { + unsafeRemoveObservation(obs.getId()); + removed.add(obs); } - }); - return result.get(); - } - - private TokenGenerator getTokenGenerator(){ - if (this.tokenGenerator == null) { - this.tokenGenerator = new RandomTokenGenerator(config.getCoapConfig()); } - return this.tokenGenerator; } - public static SingleObservation createSingleObservation(String registrationId, LwM2mPath target, ContentFormat ct, - Map ctx, String serializedObservation, TokenGenerator tokenGenerator) { - Token token = tokenGenerator.createToken(Scope.SHORT_TERM); - Map protocolData = Collections.emptyMap(); - if (serializedObservation != null) { - protocolData = new HashMap<>(); - protocolData.put(CTX_CF_OBERSATION, serializedObservation); + private boolean areTheSamePaths(Observation observation, Observation obs) { + if (observation instanceof SingleObservation && obs instanceof SingleObservation) { + return ((SingleObservation) observation).getPath().equals(((SingleObservation) obs).getPath()); } - return new SingleObservation(new ObservationIdentifier(token.getBytes()), registrationId, target, ct, ctx, protocolData); - } - - public static String createSerializedSingleObservation(JsonNode nodeSerObs, String path){ - if (nodeSerObs.has("context")){ - ((ObjectNode) nodeSerObs.get("context")).put("leshan-path", path + "\n"); - return JacksonUtil.toString(nodeSerObs); + if (observation instanceof CompositeObservation && obs instanceof CompositeObservation) { + return ((CompositeObservation) observation).getPaths().equals(((CompositeObservation) obs).getPaths()); } - return null; + return false; } @Override @@ -470,24 +377,6 @@ public class TbInMemoryRegistrationStore implements RegistrationStore, Startable return obs; } - private void cancelObservation (Observation observation, String registrationId, List removed) { - for (Observation obs : unsafeGetObservations(registrationId)) { - cancelExistingObservation(observation, obs, removed); - } - } - - private void cancelExistingObservation(Observation observation, Observation obs, List removed) { - LwM2mPath pathObservation = ((SingleObservation)observation).getPath(); - LwM2mPath pathObs = ((SingleObservation)obs).getPath(); - if ((!pathObservation.equals(pathObs) && pathObs.startWith(pathObservation)) || // pathObservation = "3", pathObs = "3/0/9" - (pathObservation.equals(pathObs) && !observation.getId().equals(obs.getId()))) { - unsafeRemoveObservation(obs.getId()); - removed.add(obs); - } else if (!pathObservation.equals(pathObs) && pathObservation.startWith(pathObs)) { // pathObservation = "3/0/9", pathObs = "3" - unsafeRemoveObservation(observation.getId()); - } - } - private void unsafeRemoveObservation(ObservationIdentifier observationId) { Observation removed = obsByToken.remove(observationId); if (removed != null) { @@ -500,7 +389,6 @@ public class TbInMemoryRegistrationStore implements RegistrationStore, Startable } } - /** * CancelAllObservation * @param registrationId diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java index b52764187d..cdd70e61a5 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.transport.lwm2m.server.store; -import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.coap.Token; import org.eclipse.californium.core.network.RandomTokenGenerator; @@ -33,10 +32,8 @@ import org.eclipse.leshan.core.observation.Observation; import org.eclipse.leshan.core.observation.ObservationIdentifier; import org.eclipse.leshan.core.observation.SingleObservation; import org.eclipse.leshan.core.peer.LwM2mIdentity; -import org.eclipse.leshan.core.request.ContentFormat; import org.eclipse.leshan.core.util.NamedThreadFactory; import org.eclipse.leshan.core.util.Validate; -import org.eclipse.leshan.server.redis.RedisRegistrationStore; import org.eclipse.leshan.server.redis.serialization.ObservationSerDes; import org.eclipse.leshan.server.redis.serialization.RegistrationSerDes; import org.eclipse.leshan.server.registration.Deregistration; @@ -45,15 +42,12 @@ import org.eclipse.leshan.server.registration.Registration; import org.eclipse.leshan.server.registration.RegistrationStore; import org.eclipse.leshan.server.registration.RegistrationUpdate; import org.eclipse.leshan.server.registration.UpdatedRegistration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.data.redis.connection.RedisClusterConnection; import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.core.Cursor; import org.springframework.data.redis.core.ScanOptions; import org.springframework.integration.redis.util.RedisLockRegistry; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.server.LwM2mVersionedModelProvider; @@ -65,19 +59,14 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.eclipse.leshan.core.californium.ObserveUtil.extractSerializedObservation; -import static org.thingsboard.server.transport.lwm2m.server.store.TbInMemoryRegistrationStore.createSerializedSingleObservation; -import static org.thingsboard.server.transport.lwm2m.server.store.TbInMemoryRegistrationStore.createSingleObservation; @Slf4j public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startable, Stoppable, Destroyable { @@ -87,8 +76,6 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab /** Defaut Extra time for registration lifetime in seconds */ public static final long DEFAULT_GRACE_PERIOD = 0; - private static final Logger LOG = LoggerFactory.getLogger(RedisRegistrationStore.class); - // Redis key prefixes public static final String REG_EP = "REG:EP:"; // (Endpoint => Registration) private static final String REG_EP_REGID_IDX = "EP:REGID:"; // secondary index key (Registration ID => Endpoint) @@ -131,7 +118,7 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab public TbLwM2mRedisRegistrationStore(LwM2MTransportServerConfig config, RedisConnectionFactory connectionFactory, long cleanPeriodInSec, long lifetimeGracePeriodInSec, int cleanLimit, LwM2mVersionedModelProvider modelProvider) { this(config, connectionFactory, Executors.newScheduledThreadPool(1, - new NamedThreadFactory(String.format("RedisRegistrationStore Cleaner (%ds)", cleanPeriodInSec))), + new NamedThreadFactory(String.format("RedisRegistrationStore Cleaner (%ds)", cleanPeriodInSec))), cleanPeriodInSec, lifetimeGracePeriodInSec, cleanLimit, modelProvider); } @@ -281,6 +268,7 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab return getRegistration(connection, registrationId); } } + private Registration getRegistration(RedisConnection connection, String registrationId) { byte[] ep = connection.get(toRegIdKey(registrationId)); if (ep == null) { @@ -494,31 +482,10 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab Lock lock = null; String lockKey = toLockKey(ep); - try { lock = redisLock.obtain(lockKey); lock.lock(); - if (observation instanceof SingleObservation) { - if (validateObserveResource(((SingleObservation)observation).getPath(), registrationId)) { - updateSingleObservation(registrationId, (SingleObservation)observation, addIfAbsent, removed, connection); - // cancel existing observations for the same path and registration id. - cancelObservation(observation, registrationId, removed, connection); - } - } else { - ContentFormat ct = ((CompositeObservation) observation).getResponseContentFormat(); - Map ctx = observation.getContext(); - String serializedObservation = extractSerializedObservation(observation); - JsonNode nodeSerObs = JacksonUtil.toJsonNode(serializedObservation); - ((CompositeObservation)observation).getPaths().forEach(path -> { - if (validateObserveResource(path, registrationId)) { - String serializedObs = createSerializedSingleObservation(nodeSerObs, path.toString()); - SingleObservation singleObservation = createSingleObservation(registrationId, path, ct, ctx, serializedObs, getTokenGenerator()); - updateSingleObservation(registrationId, singleObservation, addIfAbsent, removed, connection); - // cancel existing observations for the same path and registration id. - cancelObservation (singleObservation, registrationId, removed, connection); - } - }); - } + updateObservation(registrationId, observation, addIfAbsent, removed, connection); } finally { if (lock != null) { lock.unlock(); @@ -527,43 +494,21 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab } return removed; } - - private boolean validateObserveResource(LwM2mPath path, String registrationId){ - // check if the resource is readable. - if (path.isResource() || path.isResourceInstance()) { - ObjectModel objectModel = modelProvider.getObjectModel(getRegistration(registrationId)).getObjectModel(path.getObjectId()); - ResourceModel resourceModel = objectModel == null ? null : objectModel.resources.get(path.getResourceId()); - if (resourceModel == null) { - return false; - } else if (!resourceModel.operations.isReadable()) { - return false; - } else if (path.isResourceInstance() && !resourceModel.multiple) { - return false; - } - } - return true; - } - - private void updateSingleObservation (String registrationId, SingleObservation observation, boolean addIfAbsent, - List removed, RedisConnection connection) { + private void updateObservation(String registrationId, Observation observation, boolean addIfAbsent, + List removed, RedisConnection connection) { // Add and Get previous observation byte[] previousValue; byte[] key = toKey(OBS_TKN, observation.getId().getBytes()); byte[] serializeObs = serializeObs(observation); // we analyze the present previous value - SingleObservation existingObservation = null; - - if (addIfAbsent){ + if (addIfAbsent) { previousValue = connection.stringCommands().get(key); if (previousValue == null) { - existingObservation = validateByAbsorptionExistingObservations(observation, connection); - if (existingObservation == null){ - connection.stringCommands().set(key, serializeObs); - } else if(!existingObservation.getPath().equals(observation.getPath())) { - connection.stringCommands().set(key, serializeObs); - previousValue = serializeObs(existingObservation); - } + connection.stringCommands().set(key, serializeObs); + previousValue = serializeObs; + } else { + connection.stringCommands().set(key, serializeObs); } } else { previousValue = connection.stringCommands().getSet(key, serializeObs); @@ -573,25 +518,38 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab connection.listCommands().lPush(toKey(OBS_TKNS_REGID_IDX, registrationId), observation.getId().getBytes()); // log any collisions - if (addIfAbsent && previousValue != null) { - if (!existingObservation.getPath().equals(observation.getPath())) { - Observation previousObservation = deserializeObs(previousValue); - removed.add(previousObservation); - LOG.warn("Token collision ? observation [{}] will be replaced by observation [{}], that this observation includes input observation [{}]!", - previousObservation, observation, observation); - } else { - LOG.warn("Token collision ? existing observation [{}] includes input observation [{}]", - existingObservation, observation); + Observation previousObservation; + if (previousValue != null && previousValue.length != 0) { + previousObservation = deserializeObs(previousValue); + log.warn("Token collision ? observation [{}] will be replaced by observation [{}] ", + previousObservation, observation); + } + + // cancel existing observations for the same path and registration id. + for (Observation obs : getObservations(connection, registrationId)) { + if (areTheSamePaths(observation, obs) && !observation.getId().equals(obs.getId())) { + removed.add(obs); + unsafeRemoveObservation(connection, registrationId, obs.getId().getBytes()); } } } + private boolean areTheSamePaths(Observation observation, Observation obs) { + if (observation instanceof SingleObservation && obs instanceof SingleObservation) { + return ((SingleObservation) observation).getPath().equals(((SingleObservation) obs).getPath()); + } + if (observation instanceof CompositeObservation && obs instanceof CompositeObservation) { + return ((CompositeObservation) observation).getPaths().equals(((CompositeObservation) obs).getPaths()); + } + return false; + } @Override public Collection getObservations(String registrationId) { try (var connection = connectionFactory.getConnection()) { return getObservations(connection, registrationId); } } + @Override public Observation getObservation(String registrationId, ObservationIdentifier observationId) { return getObservations(registrationId).stream().filter( @@ -654,25 +612,6 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab return result; } - private SingleObservation validateByAbsorptionExistingObservations(SingleObservation observation, RedisConnection connection) { - LwM2mPath pathObservation = observation.getPath(); - AtomicReference result = new AtomicReference<>(); - Collection observations = getObservations(connection, observation.getRegistrationId()); - observations.stream().forEach(obs -> { - LwM2mPath pathObs = ((SingleObservation)obs).getPath(); - if ((!pathObservation.equals(pathObs) && pathObs.startWith(pathObservation)) || // pathObs = "3/0/9"-> pathObservation = "3" - (pathObservation.equals(pathObs) && !observation.getId().equals(obs.getId()))) { - result.set((SingleObservation)obs); - } else if (!pathObservation.equals(pathObs) && pathObservation.startWith(pathObs)) { // pathObs = "3" -> pathObservation = "3/0/9" - result.set(observation); - } - }); - return result.get(); - - - } - - @Override public Collection removeObservations(String registrationId) { try (var connection = connectionFactory.getConnection()) { @@ -710,7 +649,7 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab /* *************** Observation utility functions **************** */ - private TokenGenerator getTokenGenerator(){ + private TokenGenerator getTokenGenerator() { if (this.tokenGenerator == null) { this.tokenGenerator = new RandomTokenGenerator(config.getCoapConfig()); } @@ -751,8 +690,8 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab } private void cancelExistingObservation(RedisConnection connection, Observation observation, Observation obs, List removed) { - LwM2mPath pathObservation = ((SingleObservation)observation).getPath(); - LwM2mPath pathObs = ((SingleObservation)obs).getPath(); + LwM2mPath pathObservation = ((SingleObservation) observation).getPath(); + LwM2mPath pathObs = ((SingleObservation) obs).getPath(); if ((!pathObservation.equals(pathObs) && pathObs.startWith(pathObservation)) || // pathObservation = "3", pathObs = "3/0/9" (pathObservation.equals(pathObs) && !observation.getId().equals(obs.getId()))) { unsafeRemoveObservation(connection, obs.getRegistrationId(), obs.getId().getBytes()); @@ -803,7 +742,7 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab try { schedExecutor.awaitTermination(5, TimeUnit.SECONDS); } catch (InterruptedException e) { - LOG.warn("Destroying RedisRegistrationStore was interrupted.", e); + log.warn("Destroying RedisRegistrationStore was interrupted.", e); } } @@ -824,7 +763,7 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab } } } catch (Exception e) { - LOG.warn("Unexpected Exception while registration cleaning", e); + log.warn("Unexpected Exception while registration cleaning", e); } } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java index 5be27be03d..61d751d64d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java @@ -372,21 +372,19 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl LwM2mPath path = instant.getKey(); LwM2mNode node = instant.getValue(); LwM2mClient lwM2MClient = clientContext.getClientByEndpoint(registration.getEndpoint()); - String stringPath = convertObjectIdToVersionedId(path.toString(), lwM2MClient); - ObjectModel objectModelVersion = lwM2MClient.getObjectModel(stringPath, modelProvider); + ObjectModel objectModelVersion = lwM2MClient.getObjectModel(path.toString(), modelProvider); if (objectModelVersion != null) { if (node instanceof LwM2mObject) { LwM2mObject lwM2mObject = (LwM2mObject) node; - this.updateObjectResourceValue(lwM2MClient, lwM2mObject, stringPath, 0); + this.updateObjectResourceValue(lwM2MClient, lwM2mObject, path.toString(), 0); } else if (node instanceof LwM2mObjectInstance) { LwM2mObjectInstance lwM2mObjectInstance = (LwM2mObjectInstance) node; - this.updateObjectInstanceResourceValue(lwM2MClient, lwM2mObjectInstance, stringPath, 0); + this.updateObjectInstanceResourceValue(lwM2MClient, lwM2mObjectInstance, path.toString(), 0); } else if (node instanceof LwM2mResource) { LwM2mResource lwM2mResource = (LwM2mResource) node; - this.updateResourcesValue(lwM2MClient, lwM2mResource, stringPath, Mode.UPDATE, 0); + this.updateResourcesValue(lwM2MClient, lwM2mResource, path.toString(), Mode.UPDATE, 0); } } - tryAwake(lwM2MClient); } } @@ -569,7 +567,6 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl } private void updateObjectInstanceResourceValue(LwM2mClient client, LwM2mObjectInstance lwM2mObjectInstance, String pathIdVer, int code) { - LwM2mPath pathIds = new LwM2mPath(fromVersionedIdToObjectId(pathIdVer)); lwM2mObjectInstance.getResources().forEach((resourceId, resource) -> { String pathRez = pathIdVer + "/" + resourceId; this.updateResourcesValue(client, resource, pathRez, Mode.UPDATE, code); @@ -584,11 +581,12 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl * #4 updateAttrTelemetry * @param lwM2MClient - Registration LwM2M Client * @param lwM2mResource - LwM2mSingleResource response.getContent() - * @param path - resource + * @param stringPath - resource * @param mode - Replace, Update */ - private void updateResourcesValue(LwM2mClient lwM2MClient, LwM2mResource lwM2mResource, String path, Mode mode, int code) { + private void updateResourcesValue(LwM2mClient lwM2MClient, LwM2mResource lwM2mResource, String stringPath, Mode mode, int code) { Registration registration = lwM2MClient.getRegistration(); + String path = convertObjectIdToVersionedId(stringPath, lwM2MClient); if (lwM2MClient.saveResourceValue(path, lwM2mResource, modelProvider, mode)) { if (path.equals(convertObjectIdToVersionedId(FW_NAME_ID, lwM2MClient))) { otaService.onCurrentFirmwareNameUpdate(lwM2MClient, (String) lwM2mResource.getValue()); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java index e2e31f2768..6b90ae21b6 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/utils/LwM2MTransportUtil.java @@ -156,21 +156,19 @@ public class LwM2MTransportUtil { } public static String convertObjectIdToVersionedId(String path, LwM2mClient lwM2MClient) { - String ver = String.valueOf(lwM2MClient.getSupportedObjectVersion(new LwM2mPath(path).getObjectId())); - return convertObjectIdToVerId(path, ver); - } - public static String convertObjectIdToVerId(String path, String ver) { - ver = ver != null ? ver : TbLwM2mVersion.VERSION_1_0.getVersion().toString(); - try { - String[] keyArray = path.split(LWM2M_SEPARATOR_PATH); - if (keyArray.length > 1) { - keyArray[1] = keyArray[1] + LWM2M_SEPARATOR_KEY + ver; + String[] keyArray = path.split(LWM2M_SEPARATOR_PATH); + if (keyArray.length > 1) { + try { + Integer objectId = Integer.valueOf((keyArray[1].split(LWM2M_SEPARATOR_KEY))[0]); + String ver = String.valueOf(lwM2MClient.getSupportedObjectVersion(objectId)); + ver = ver != null ? ver : TbLwM2mVersion.VERSION_1_0.getVersion().toString(); + keyArray[1] = String.valueOf(keyArray[1]).contains(LWM2M_SEPARATOR_KEY) ? keyArray[1] : keyArray[1] + LWM2M_SEPARATOR_KEY + ver; return StringUtils.join(keyArray, LWM2M_SEPARATOR_PATH); - } else { - return path; + } catch (Exception e) { + return null; } - } catch (Exception e) { - return null; + } else { + return path; } } @@ -214,20 +212,20 @@ public class LwM2MTransportUtil { } public static Map convertMultiResourceValuesFromRpcBody(Object value, ResourceModel.Type type, String versionedId) throws Exception { - String valueJsonStr = JacksonUtil.toString(value); - JsonElement element = JsonUtils.parse(valueJsonStr); - return convertMultiResourceValuesFromJson(element, type, versionedId); + String valueJsonStr = JacksonUtil.toString(value); + JsonElement element = JsonUtils.parse(valueJsonStr); + return convertMultiResourceValuesFromJson(element, type, versionedId); } public static Map convertMultiResourceValuesFromJson(JsonElement newValProto, ResourceModel.Type type, String versionedId) { Map newValues = new HashMap<>(); newValProto.getAsJsonObject().entrySet().forEach((obj) -> { - newValues.put(Integer.valueOf(obj.getKey()), convertValueByTypeResource (obj.getValue().getAsString(), type, versionedId)); + newValues.put(Integer.valueOf(obj.getKey()), convertValueByTypeResource(obj.getValue().getAsString(), type, versionedId)); }); return newValues; } - public static Object convertValueByTypeResource (String value, ResourceModel.Type type, String versionedId) { + public static Object convertValueByTypeResource(String value, ResourceModel.Type type, String versionedId) { return LwM2mValueConverterImpl.getInstance().convertValue(value, STRING, type, new LwM2mPath(fromVersionedIdToObjectId(versionedId))); } @@ -356,7 +354,7 @@ public class LwM2MTransportUtil { serverCoapConfig.setTransient(DTLS_CONNECTION_ID_LENGTH); serverCoapConfig.setTransient(DTLS_CONNECTION_ID_NODE_ID); serverCoapConfig.set(DTLS_CONNECTION_ID_LENGTH, cIdLength); - if ( cIdLength > 4) { + if (cIdLength > 4) { serverCoapConfig.set(DTLS_CONNECTION_ID_NODE_ID, 0); } else { serverCoapConfig.set(DTLS_CONNECTION_ID_NODE_ID, null); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index c76627d3a1..87d466e11a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -16,6 +16,7 @@ package org.thingsboard.server.msa; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.restassured.RestAssured; import io.restassured.common.mapper.TypeRef; import io.restassured.config.HeaderConfig; @@ -116,6 +117,16 @@ public class TestRestClient { .as(Device.class); } + public ObjectNode postRpcLwm2mParams(String deviceIdStr, String body) { + return given().spec(requestSpec).body(body) + .post("/api/plugins/rpc/twoway/" + deviceIdStr) + .then() + .statusCode(HTTP_OK) + .extract() + .as(ObjectNode.class); + } + + public Device getDeviceByName(String deviceName) { return given().spec(requestSpec).pathParam("deviceName", deviceName) .get("/api/tenant/devices?deviceName={deviceName}") diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractLwm2mClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/AbstractLwm2mClientTest.java similarity index 50% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractLwm2mClientTest.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/AbstractLwm2mClientTest.java index c0ae8e9776..b60eb15b5c 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractLwm2mClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/AbstractLwm2mClientTest.java @@ -13,12 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa; +package org.thingsboard.server.msa.connectivity.lwm2m; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.eclipse.leshan.client.object.Security; +import org.eclipse.leshan.core.ResponseCode; import org.eclipse.leshan.core.util.Hex; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Device; @@ -42,12 +48,14 @@ import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTrans import org.thingsboard.server.common.data.device.profile.lwm2m.OtherConfiguration; import org.thingsboard.server.common.data.device.profile.lwm2m.TelemetryMappingConfiguration; import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBootstrapServerCredential; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; -import org.thingsboard.server.msa.connectivity.lwm2m.LwM2MTestClient; -import org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState; +import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.connectivity.lwm2m.client.LwM2MTestClient; +import org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState; import java.net.ServerSocket; import java.nio.charset.StandardCharsets; @@ -55,6 +63,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -63,45 +72,45 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.eclipse.leshan.client.object.Security.psk; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.CLIENT_ENDPOINT_NO_SEC; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.CLIENT_ENDPOINT_PSK; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.CLIENT_LWM2M_SETTINGS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.CLIENT_PSK_IDENTITY; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.CLIENT_PSK_KEY; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_INIT; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_STARTED; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_SUCCESS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.OBSERVE_ATTRIBUTES_WITHOUT_PARAMS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.SECURE_URI; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.SECURITY_NO_SEC; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.resources; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.shortServerId; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.testng.AssertJUnit.assertEquals; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.CLIENT_ENDPOINT_PSK; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.CLIENT_LWM2M_SETTINGS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.CLIENT_PSK_IDENTITY; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.CLIENT_PSK_KEY; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_INIT; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_STARTED; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_SUCCESS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.OBSERVE_ATTRIBUTES_WITH_PARAMS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.SECURE_URI; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.SECURITY_NO_SEC; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.resources; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.shortServerId; @Slf4j -public class AbstractLwm2mClientTest extends AbstractContainerTest{ +public class AbstractLwm2mClientTest extends AbstractContainerTest { protected ScheduledExecutorService executor; protected Security security; protected final PageLink pageLink = new PageLink(30); protected TenantId tenantId; - protected DeviceProfile lwm2mDeviceProfile; - protected Device lwM2MDeviceTest; - protected LwM2MTestClient lwM2MTestClient; public final Set expectedStatusesRegistrationLwm2mSuccess = new HashSet<>(Arrays.asList(ON_INIT, ON_REGISTRATION_STARTED, ON_REGISTRATION_SUCCESS)); - public void connectLwm2mClientNoSec() throws Exception { - LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(CLIENT_ENDPOINT_NO_SEC)); - basicTestConnection(SECURITY_NO_SEC, - deviceCredentials, - CLIENT_ENDPOINT_NO_SEC, - "TestConnection Lwm2m NoSec (msa)"); + public void createLwm2mDevicesForConnectNoSec(String name, Lwm2mDevicesForTest devicesForTest) throws Exception { + String clientEndpoint = name + "-" + RandomStringUtils.randomAlphanumeric(7); + LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(clientEndpoint)); + Device lwM2MDeviceTest = createDeviceWithCredentials(deviceCredentials, clientEndpoint, devicesForTest.getLwm2mDeviceProfile().getId()); + LwM2MTestClient lwM2MTestClient = createNewClient(SECURITY_NO_SEC, clientEndpoint, executor); + devicesForTest.setLwM2MDeviceTest(lwM2MDeviceTest); + devicesForTest.setLwM2MTestClient(lwM2MTestClient); } - public void connectLwm2mClientPsk() throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_PSK; + public void createLwm2mDevicesForConnectPsk(Lwm2mDevicesForTest devicesForTest) throws Exception { + String clientEndpoint = CLIENT_ENDPOINT_PSK +"-" + RandomStringUtils.randomAlphanumeric(7); String identity = CLIENT_PSK_IDENTITY; String keyPsk = CLIENT_PSK_KEY; PSKClientCredential clientCredentials = new PSKClientCredential(); @@ -113,40 +122,64 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ identity.getBytes(StandardCharsets.UTF_8), Hex.decodeHex(keyPsk.toCharArray())); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsSecurePsk(clientCredentials); + Device lwM2MDeviceTest = createDeviceWithCredentials(deviceCredentials, clientEndpoint, devicesForTest.getLwm2mDeviceProfile().getId()); + LwM2MTestClient lwM2MTestClient = createNewClient(security, clientEndpoint, executor); + devicesForTest.setLwM2MDeviceTest(lwM2MDeviceTest); + devicesForTest.setLwM2MTestClient(lwM2MTestClient); + } - basicTestConnection(security, - deviceCredentials, - clientEndpoint, - "TestConnection Lwm2m Rpc (msa)"); + public void observeResource_Update_AfterUpdateRegistration_test(LwM2MTestClient lwM2MTestClient, String deviceIdStr) throws Exception { + awaitUpdateRegistrationSuccess(lwM2MTestClient, 5); + sendCancelObserveAllWithAwait(deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); + String param = "/3_1.2/0/9"; + sendRpcObserveWithContainsLwM2mSingleResource(param, deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); + sendCancelObserveAllWithAwait(deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); + sendRpcObserveWithContainsLwM2mSingleResource(param, deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 2); + } + public void observeCompositeResource_Update_AfterUpdateRegistration_test(LwM2MTestClient lwM2MTestClient, String deviceIdStr) throws Exception { + awaitUpdateRegistrationSuccess(lwM2MTestClient, 5); + sendCancelObserveAllWithAwait(deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); + String expectedKey3_0_9 = "batteryLevel"; +// String expectedKey3_0_14 = "UtfOffset"; +// String expectedKey19_0_0 = "dataRead"; +// String expectedKey19_1_0 = "dataWrite"; +// String expectedKeys = "[\"" + expectedKey3_0_9 + "\", \"" + expectedKey3_0_14 + "\", \"" + expectedKey19_0_0 + "\", \"" + expectedKey19_1_0 + "\", \"" + expectedKey3_0_9 + "\"]"; + String expectedKeys = "[\"" + expectedKey3_0_9 + "\"]"; + sendRpcObserveCompositeWithContainsLwM2mSingleResource(expectedKeys, deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); + sendCancelObserveAllWithAwait(deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); + sendRpcObserveCompositeWithContainsLwM2mSingleResource(expectedKeys, deviceIdStr); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 2); } - public void basicTestConnection(Security security, - LwM2MDeviceCredentials deviceCredentials, - String clientEndpoint, String alias) throws Exception { - // create lwm2mClient and lwM2MDevice - lwM2MDeviceTest = createDeviceWithCredentials(deviceCredentials, clientEndpoint); - lwM2MTestClient = createNewClient(security, clientEndpoint, executor); + public void basicTestConnection(LwM2MTestClient lwM2MTestClient, String alias) throws Exception { + LwM2MClientState finishState = ON_REGISTRATION_SUCCESS; + await(alias + " - " + ON_REGISTRATION_STARTED) + .atMost(40, TimeUnit.SECONDS) + .until(() -> { + log.warn("msa basicTestConnection started -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); + return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_REGISTRATION_STARTED); + }); + await(alias + " - " + ON_UPDATE_SUCCESS) + .atMost(40, TimeUnit.SECONDS) + .until(() -> { + log.warn("msa basicTestConnection update -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); + return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_UPDATE_SUCCESS); + }); + assertThat(lwM2MTestClient.getClientStates()).containsAll(expectedStatusesRegistrationLwm2mSuccess); - LwM2MClientState finishState = ON_REGISTRATION_SUCCESS; - await(alias + " - " + ON_REGISTRATION_STARTED) - .atMost(40, TimeUnit.SECONDS) - .until(() -> { - log.warn("msa basicTestConnection started -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); - return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_REGISTRATION_STARTED); - }); - await(alias + " - " + ON_UPDATE_SUCCESS) - .atMost(40, TimeUnit.SECONDS) - .until(() -> { - log.warn("msa basicTestConnection update -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); - return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_UPDATE_SUCCESS); - }); - assertThat(lwM2MTestClient.getClientStates()).containsAll(expectedStatusesRegistrationLwm2mSuccess); } public LwM2MTestClient createNewClient(Security security, String endpoint, ScheduledExecutorService executor) throws Exception { this.executor = executor; - LwM2MTestClient lwM2MTestClient = new LwM2MTestClient(endpoint); + LwM2MTestClient lwM2MTestClient = new LwM2MTestClient(executor, endpoint); try (ServerSocket socket = new ServerSocket(0)) { int clientPort = socket.getLocalPort(); lwM2MTestClient.init(security, clientPort); @@ -154,16 +187,16 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ return lwM2MTestClient; } - protected void destroyAfter(){ - clientDestroy(); - deviceDestroy(); - deviceProfileDestroy(); + protected void destroyAfter(Lwm2mDevicesForTest devicesForTest){ + clientDestroy(devicesForTest.getLwM2MTestClient()); + deviceDestroy(devicesForTest.getLwM2MDeviceTest()); + deviceProfileDestroy(devicesForTest.getLwm2mDeviceProfile()); if (executor != null) { executor.shutdown(); } } - protected void clientDestroy() { + protected void clientDestroy(LwM2MTestClient lwM2MTestClient) { try { if (lwM2MTestClient != null) { lwM2MTestClient.destroy(); @@ -172,7 +205,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ log.error("Failed client Destroy", e); } } - protected void deviceDestroy() { + protected void deviceDestroy(Device lwM2MDeviceTest) { try { if (lwM2MDeviceTest != null) { testRestClient.deleteDeviceIfExists(lwM2MDeviceTest.getId()); @@ -182,13 +215,13 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ } } - protected void initTest(String deviceProfileName) throws Exception { + protected DeviceProfile initTest(String deviceProfileName) throws Exception { if (executor != null) { executor.shutdown(); } executor = Executors.newScheduledThreadPool(10, ThingsBoardThreadFactory.forName("test-scheduled-" + deviceProfileName)); - lwm2mDeviceProfile = getDeviceProfile(deviceProfileName); + DeviceProfile lwm2mDeviceProfile = getDeviceProfile(deviceProfileName); tenantId = lwm2mDeviceProfile.getTenantId(); for (String resourceName : resources) { @@ -201,6 +234,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ lwModel.setData(bytes); testRestClient.postTbResourceIfNotExists(lwModel); } + return lwm2mDeviceProfile; } protected DeviceProfile getDeviceProfile(String deviceProfileName) throws Exception { @@ -235,7 +269,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ return deviceProfile; } - protected void deviceProfileDestroy(){ + protected void deviceProfileDestroy(DeviceProfile lwm2mDeviceProfile){ try { if (lwm2mDeviceProfile != null) { testRestClient.deleteDeviceProfileIfExists(lwm2mDeviceProfile); @@ -245,17 +279,17 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ } } - protected Device createDeviceWithCredentials(LwM2MDeviceCredentials deviceCredentials, String clientEndpoint) throws Exception { - Device device = createDevice(deviceCredentials, clientEndpoint); + protected Device createDeviceWithCredentials(LwM2MDeviceCredentials deviceCredentials, String clientEndpoint, DeviceProfileId profileId) throws Exception { + Device device = createDevice(deviceCredentials, clientEndpoint, profileId); return device; } - protected Device createDevice(LwM2MDeviceCredentials credentials, String clientEndpoint) throws Exception { + protected Device createDevice(LwM2MDeviceCredentials credentials, String clientEndpoint, DeviceProfileId profileId) throws Exception { Device device = testRestClient.getDeviceByNameIfExists(clientEndpoint); if (device == null) { device = new Device(); device.setName(clientEndpoint); - device.setDeviceProfileId(lwm2mDeviceProfile.getId()); + device.setDeviceProfileId(profileId); device.setTenantId(tenantId); device = testRestClient.postDevice("", device); } @@ -288,7 +322,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ protected Lwm2mDeviceProfileTransportConfiguration getTransportConfiguration() { List bootstrapServerCredentials = new ArrayList<>(); Lwm2mDeviceProfileTransportConfiguration transportConfiguration = new Lwm2mDeviceProfileTransportConfiguration(); - TelemetryMappingConfiguration observeAttrConfiguration = JacksonUtil.fromString(OBSERVE_ATTRIBUTES_WITHOUT_PARAMS, TelemetryMappingConfiguration.class); + TelemetryMappingConfiguration observeAttrConfiguration = JacksonUtil.fromString(OBSERVE_ATTRIBUTES_WITH_PARAMS, TelemetryMappingConfiguration.class); OtherConfiguration clientLwM2mSettings = JacksonUtil.fromString(CLIENT_LWM2M_SETTINGS, OtherConfiguration.class); transportConfiguration.setBootstrapServerUpdateEnable(true); transportConfiguration.setObserveAttr(observeAttrConfiguration); @@ -317,4 +351,75 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest{ bootstrapCredentials.setLwm2mServer(serverCredentials); return bootstrapCredentials; } + + protected void sendCancelObserveAllWithAwait(String deviceIdStr) throws Exception { + ObjectNode rpcActualResultCancelAll = sendRpcObserve("ObserveCancelAll", null, deviceIdStr); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultCancelAll.get("result").asText()); + awaitObserveReadAll(0, deviceIdStr); + } + + protected void awaitObserveReadAll(int cntObserve, String deviceIdStr) throws Exception { + await("ObserveReadAll after start client/test: countObserve " + cntObserve) + .atMost(40, TimeUnit.SECONDS) + .until(() -> cntObserve == getCntObserveAll(deviceIdStr)); + } + protected void awaitUpdateRegistrationSuccess(LwM2MTestClient lwM2MTestClient, int cntUpdate) throws Exception { + cntUpdate = cntUpdate + lwM2MTestClient.getCountUpdateRegistrationSuccess(); + int finalCntUpdate = cntUpdate; + await("Update Registration client: countUpdateSuccess " + finalCntUpdate) + .atMost(40, TimeUnit.SECONDS) + .until(() -> finalCntUpdate <= lwM2MTestClient.getCountUpdateRegistrationSuccess()); + } + protected void awaitObserveReadResource_3_0_9(int cntRead, String deviceIdStr) throws Exception { + await("Read value 3/0/9 after start observe: countRead " + cntRead) + .atMost(40, TimeUnit.SECONDS) + .until(() -> cntRead == getCntObserveAll(deviceIdStr)); + } + + protected Integer getCntObserveAll(String deviceIdStr) throws Exception { + ObjectNode rpcActualResultBefore = sendRpcObserve("ObserveReadAll", null, deviceIdStr); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultBefore.get("result").asText()); + JsonElement element = JsonParser.parseString(rpcActualResultBefore.get("value").asText()); + return element.isJsonArray() ? ((JsonArray)element).size() : null; + } + + private void sendRpcObserveWithContainsLwM2mSingleResource(String params, String deviceIdStr) throws Exception { + String rpcActualResult = sendRpcObserveWithResultValue(params, deviceIdStr); + assertTrue(rpcActualResult.contains("LwM2mSingleResource")); + assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceIdStr)).get()); + } + + private void sendRpcObserveCompositeWithContainsLwM2mSingleResource(String params, String deviceIdStr) throws Exception { + String rpcActualResult = sendRpcObserveCompositeWithResultValue(params, deviceIdStr); + assertTrue(rpcActualResult.contains("LwM2mSingleResource")); + assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceIdStr)).get()); + } + + private String sendRpcObserveWithResultValue(String params, String deviceIdStr) throws Exception { + ObjectNode rpcActualResult = sendRpcObserve("Observe", params, deviceIdStr); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + return rpcActualResult.get("value").asText(); + } + + private String sendRpcObserveCompositeWithResultValue(String params, String deviceIdStr) throws Exception { + ObjectNode rpcActualResult = sendRpcObserveComposite(params, deviceIdStr); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + return rpcActualResult.get("value").asText(); + } + + protected ObjectNode sendRpcObserve(String method, String params, String deviceIdStr) throws Exception { + String sendRpcRequest; + if (params == null) { + sendRpcRequest = "{\"method\": \"" + method + "\"}"; + } + else { + sendRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"id\": \"" + params + "\"}}"; + } + return testRestClient.postRpcLwm2mParams(deviceIdStr, sendRpcRequest); + } + protected ObjectNode sendRpcObserveComposite(String keys, String deviceIdStr) throws Exception { + String method = "ObserveComposite"; + String sendRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"keys\":" + keys + "}}"; + return testRestClient.postRpcLwm2mParams(deviceIdStr, sendRpcRequest); + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java new file mode 100644 index 0000000000..5374449e11 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.msa.connectivity.lwm2m; + + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.msa.connectivity.lwm2m.client.LwM2MTestClient; + +@Slf4j +@Data +public class Lwm2mDevicesForTest { + + Device lwM2MDeviceTest; + LwM2MTestClient lwM2MTestClient; + DeviceProfile lwm2mDeviceProfile; + public Lwm2mDevicesForTest(DeviceProfile deviceProfile) { + this.lwm2mDeviceProfile = deviceProfile; + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/FwLwM2MDevice.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/FwLwM2MDevice.java similarity index 98% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/FwLwM2MDevice.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/FwLwM2MDevice.java index 2910a2b7da..0037cff467 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/FwLwM2MDevice.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/FwLwM2MDevice.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa.connectivity.lwm2m; +package org.thingsboard.server.msa.connectivity.lwm2m.client; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.client.resource.BaseInstanceEnabler; diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/LwM2MTestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2MTestClient.java similarity index 79% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/LwM2MTestClient.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2MTestClient.java index c9f193704b..9fb9d4c67d 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/LwM2MTestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2MTestClient.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa.connectivity.lwm2m; +package org.thingsboard.server.msa.connectivity.lwm2m.client; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -44,13 +44,14 @@ import org.eclipse.leshan.core.model.LwM2mModel; import org.eclipse.leshan.core.model.ObjectLoader; import org.eclipse.leshan.core.model.ObjectModel; import org.eclipse.leshan.core.model.StaticModel; +import org.eclipse.leshan.core.node.LwM2mPath; import org.eclipse.leshan.core.node.codec.DefaultLwM2mDecoder; import org.eclipse.leshan.core.node.codec.DefaultLwM2mEncoder; import org.eclipse.leshan.core.request.BootstrapRequest; import org.eclipse.leshan.core.request.DeregisterRequest; import org.eclipse.leshan.core.request.RegisterRequest; import org.eclipse.leshan.core.request.UpdateRequest; -import org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState; +import org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState; import java.io.IOException; import java.net.InetSocketAddress; @@ -60,6 +61,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -70,33 +72,34 @@ import static org.eclipse.leshan.core.LwM2mId.DEVICE; import static org.eclipse.leshan.core.LwM2mId.FIRMWARE; import static org.eclipse.leshan.core.LwM2mId.SECURITY; import static org.eclipse.leshan.core.LwM2mId.SERVER; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_FAILURE; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_STARTED; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_SUCCESS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_TIMEOUT; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_FAILURE; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_STARTED; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_SUCCESS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_TIMEOUT; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_EXPECTED_ERROR; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_INIT; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_FAILURE; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_STARTED; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_TIMEOUT; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_FAILURE; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_STARTED; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_SUCCESS; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_TIMEOUT; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.resources; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.serverId; -import static org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mTestHelper.shortServerId; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_FAILURE; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_STARTED; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_SUCCESS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_TIMEOUT; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_FAILURE; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_STARTED; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_SUCCESS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_DEREGISTRATION_TIMEOUT; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_EXPECTED_ERROR; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_INIT; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_FAILURE; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_STARTED; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_TIMEOUT; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_FAILURE; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_STARTED; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_SUCCESS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_TIMEOUT; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.resources; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.serverId; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.shortServerId; @Slf4j @Data public class LwM2MTestClient { + private final ScheduledExecutorService executor; private final String endpoint; private LeshanClient leshanClient; private SimpleLwM2MDevice lwM2MDevice; @@ -106,6 +109,9 @@ public class LwM2MTestClient { private FwLwM2MDevice fwLwM2MDevice; private Map clientDtlsCid; + private int countUpdateRegistrationSuccess; + private int countReadObserveAfterUpdateRegistrationSuccess; + public void init(Security security, int clientPort) throws InvalidDDFFileException, IOException { assertThat(leshanClient).as("client already initialized").isNull(); @@ -123,7 +129,8 @@ public class LwM2MTestClient { lwm2mServer.setId(serverId); initializer.setInstancesForObject(SERVER, lwm2mServer); - initializer.setInstancesForObject(DEVICE, lwM2MDevice = new SimpleLwM2MDevice()); + SimpleLwM2MDevice simpleLwM2MDevice = new SimpleLwM2MDevice(executor); + initializer.setInstancesForObject(DEVICE, lwM2MDevice = simpleLwM2MDevice); initializer.setClassForObject(ACCESS_CONTROL, DummyInstanceEnabler.class); initializer.setInstancesForObject(FIRMWARE, fwLwM2MDevice = new FwLwM2MDevice()); @@ -218,6 +225,7 @@ public class LwM2MTestClient { clientDtlsCid = new HashMap<>(); clientStates.add(ON_INIT); leshanClient = builder.build(); + simpleLwM2MDevice.setLwM2MTestClient(this); LwM2mClientObserver observer = new LwM2mClientObserver() { @Override @@ -248,7 +256,7 @@ public class LwM2MTestClient { @Override public void onRegistrationSuccess(LwM2mServer server, RegisterRequest request, String registrationID) { clientStates.add(ON_REGISTRATION_SUCCESS); - } + } @Override public void onRegistrationFailure(LwM2mServer server, RegisterRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { @@ -268,6 +276,8 @@ public class LwM2MTestClient { @Override public void onUpdateSuccess(LwM2mServer server, UpdateRequest request) { clientStates.add(ON_UPDATE_SUCCESS); + countUpdateRegistrationSuccess++; + countReadObserveAfterUpdateRegistrationSuccess = 0; } @Override @@ -319,6 +329,12 @@ public class LwM2MTestClient { public void objectAdded(LwM2mObjectEnabler object) { log.info("Object {} v{} enabled.", object.getId(), object.getObjectModel().version); } + + @Override + public void resourceChanged(LwM2mPath... paths) { + countReadObserveAfterUpdateRegistrationSuccess++; + log.trace("resourceChanged paths {} cntReadObserve {} cntUpdateSuccess {} .", paths, countReadObserveAfterUpdateRegistrationSuccess, countUpdateRegistrationSuccess); + } }); leshanClient.start(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mBinaryAppDataContainer.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mBinaryAppDataContainer.java new file mode 100644 index 0000000000..d8f598fa93 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mBinaryAppDataContainer.java @@ -0,0 +1,230 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.connectivity.lwm2m.client; + +import lombok.extern.slf4j.Slf4j; +import org.eclipse.leshan.client.resource.BaseInstanceEnabler; +import org.eclipse.leshan.client.servers.LwM2mServer; +import org.eclipse.leshan.core.model.ObjectModel; +import org.eclipse.leshan.core.model.ResourceModel; +import org.eclipse.leshan.core.node.LwM2mMultipleResource; +import org.eclipse.leshan.core.node.LwM2mResource; +import org.eclipse.leshan.core.response.ReadResponse; +import org.eclipse.leshan.core.response.WriteResponse; + +import javax.security.auth.Destroyable; +import java.sql.Time; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +@Slf4j +public class LwM2mBinaryAppDataContainer extends BaseInstanceEnabler implements Destroyable { + + /** + * id = 0 + * Multiple + * base64 + */ + + /** + * Example1: + * InNlcnZpY2VJZCI6Ik1ldGVyIiwNCiJzZXJ2aWNlRGF0YSI6ew0KImN1cnJlbnRSZWFka + * W5nIjoiNDYuMyIsDQoic2lnbmFsU3RyZW5ndGgiOjE2LA0KImRhaWx5QWN0aXZpdHlUaW1lIjo1NzA2DQo= + * "serviceId":"Meter", + * "serviceData":{ + * "currentReading":"46.3", + * "signalStrength":16, + * "dailyActivityTime":5706 + */ + + /** + * Example2: + * InNlcnZpY2VJZCI6IldhdGVyTWV0ZXIiLA0KImNtZCI6IlNFVF9URU1QRVJBVFVSRV9SRUFEX + * 1BFUklPRCIsDQoicGFyYXMiOnsNCiJ2YWx1ZSI6NA0KICAgIH0sDQoNCg0K + * "serviceId":"WaterMeter", + * "cmd":"SET_TEMPERATURE_READ_PERIOD", + * "paras":{ + * "value":4 + * }, + */ + + Map data; + private Integer priority = 0; + private Time timestamp; + private String description; + private String dataFormat; + private Integer appID = -1; + private static final List supportedResources = Arrays.asList(0, 1, 2, 3, 4, 5); + + public LwM2mBinaryAppDataContainer() { + } + + public LwM2mBinaryAppDataContainer(ScheduledExecutorService executorService, Integer id) { + try { + if (id != null) this.setId(id); + executorService.scheduleWithFixedDelay(() -> { + fireResourceChange(0); + fireResourceChange(2); + } + , 1800000, 1800000, TimeUnit.MILLISECONDS); // 30 MIN + } catch (Throwable e) { + log.error("[{}]Throwable", e.toString()); + e.printStackTrace(); + } + } + + @Override + public ReadResponse read(LwM2mServer identity, int resourceId) { + try { + switch (resourceId) { + case 0: + ReadResponse response = ReadResponse.success(resourceId, getData(), ResourceModel.Type.OPAQUE); + return response; + case 1: + return ReadResponse.success(resourceId, getPriority()); + case 2: + return ReadResponse.success(resourceId, getTimestamp()); + case 3: + return ReadResponse.success(resourceId, getDescription()); + case 4: + return ReadResponse.success(resourceId, getDataFormat()); + case 5: + return ReadResponse.success(resourceId, getAppID()); + default: + return super.read(identity, resourceId); + } + } catch (Exception e) { + return ReadResponse.badRequest(e.getMessage()); + } + } + + @Override + public WriteResponse write(LwM2mServer identity, boolean replace, int resourceId, LwM2mResource value) { + log.info("Write on Device resource /[{}]/[{}]/[{}]", getModel().id, getId(), resourceId); + switch (resourceId) { + case 0: + if (setData(value, replace)) { + return WriteResponse.success(); + } else { + WriteResponse.badRequest("Invalidate value ..."); + } + case 1: + setPriority((Integer) (value.getValue() instanceof Long ? ((Long) value.getValue()).intValue() : value.getValue())); + fireResourceChange(resourceId); + return WriteResponse.success(); + case 2: + setTimestamp(((Date) value.getValue()).getTime()); + fireResourceChange(resourceId); + return WriteResponse.success(); + case 3: + setDescription((String) value.getValue()); + fireResourceChange(resourceId); + return WriteResponse.success(); + case 4: + setDataFormat((String) value.getValue()); + fireResourceChange(resourceId); + return WriteResponse.success(); + case 5: + setAppID((Integer) value.getValue()); + fireResourceChange(resourceId); + return WriteResponse.success(); + default: + return super.write(identity, replace, resourceId, value); + } + } + + private Integer getAppID() { + return this.appID; + } + + private void setAppID(Integer appId) { + this.appID = appId; + } + + private void setDataFormat(String value) { + this.dataFormat = value; + } + + private String getDataFormat() { + return this.dataFormat == null ? "OPAQUE" : this.dataFormat; + } + + private void setDescription(String value) { + this.description = value; + } + + private String getDescription() { +// return this.description == null ? "meter reading" : this.description; + return this.description; + } + + private void setTimestamp(long time) { + this.timestamp = new Time(time); + } + + private Time getTimestamp() { + return this.timestamp != null ? this.timestamp : new Time(new Date().getTime()); + } + + private boolean setData(LwM2mResource value, boolean replace) { + try { + if (value instanceof LwM2mMultipleResource) { + if (replace || this.data == null) { + this.data = new HashMap<>(); + } + value.getInstances().values().forEach(v -> { + this.data.put(v.getId(), (byte[]) v.getValue()); + }); + return true; + } else { + return false; + } + } catch (Exception e) { + return false; + } + } + + private Map getData() { + if (data == null) { + this.data = new HashMap<>(); + this.data.put(0, new byte[]{(byte) 0xAC}); + } + return data; + } + + @Override + public List getAvailableResourceIds(ObjectModel model) { + return supportedResources; + } + + @Override + public void destroy() { + } + + private int getPriority() { + return this.priority; + } + + private void setPriority(int value) { + this.priority = value; + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mTemperatureSensor.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mTemperatureSensor.java new file mode 100644 index 0000000000..2ad6fdc8d1 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mTemperatureSensor.java @@ -0,0 +1,190 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.connectivity.lwm2m.client; + +import lombok.extern.slf4j.Slf4j; +import org.eclipse.leshan.client.LeshanClient; +import org.eclipse.leshan.client.resource.BaseInstanceEnabler; +import org.eclipse.leshan.client.send.ManualDataSender; +import org.eclipse.leshan.client.servers.LwM2mServer; +import org.eclipse.leshan.core.model.ObjectModel; +import org.eclipse.leshan.core.node.LwM2mPath; +import org.eclipse.leshan.core.request.ContentFormat; +import org.eclipse.leshan.core.request.argument.Arguments; +import org.eclipse.leshan.core.response.ExecuteResponse; +import org.eclipse.leshan.core.response.ReadResponse; + +import javax.security.auth.Destroyable; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class LwM2mTemperatureSensor extends BaseInstanceEnabler implements Destroyable { + + private static final String UNIT_CELSIUS = "cel"; + private double currentTemp = 20d; + private double minMeasuredValue = currentTemp; + private double maxMeasuredValue = currentTemp; + + private LeshanClient leshanClient; + private List containingValues; + protected static final Random RANDOM = new Random(); + private static final List supportedResources = Arrays.asList(5601, 5602, 5700, 5701); + + public LwM2mTemperatureSensor() { + + } + + public LwM2mTemperatureSensor(ScheduledExecutorService executorService, Integer id) { + try { + if (id != null) this.setId(id); + executorService.scheduleWithFixedDelay(this::adjustTemperature, 2000, 2000, TimeUnit.MILLISECONDS); + } catch (Throwable e) { + log.error("[{}]Throwable", e.toString()); + e.printStackTrace(); + } + } + + @Override + public synchronized ReadResponse read(LwM2mServer identity, int resourceId) { + log.info("Read on Temperature resource /[{}]/[{}]/[{}]", getModel().id, getId(), resourceId); + switch (resourceId) { + case 5601: + return ReadResponse.success(resourceId, getTwoDigitValue(minMeasuredValue)); + case 5602: + return ReadResponse.success(resourceId, getTwoDigitValue(maxMeasuredValue)); + case 5700: + if (identity == LwM2mServer.SYSTEM) { + setTemperature(); + setData(); + return ReadResponse.success(resourceId, getTwoDigitValue(currentTemp)); + } else if (this.getId() == 12 && this.leshanClient != null) { + containingValues = new ArrayList<>(); + sendCollected(5700); + return ReadResponse.success(resourceId, getData()); + } else { + return ReadResponse.success(resourceId, getTwoDigitValue(currentTemp)); + } + case 5701: + return ReadResponse.success(resourceId, UNIT_CELSIUS); + default: + return super.read(identity, resourceId); + } + } + + @Override + public synchronized ExecuteResponse execute(LwM2mServer identity, int resourceId, Arguments arguments) { + log.info("Execute on Temperature resource /[{}]/[{}]/[{}]", getModel().id, getId(), resourceId); + switch (resourceId) { + case 5605: + resetMinMaxMeasuredValues(); + return ExecuteResponse.success(); + default: + return super.execute(identity, resourceId, arguments); + } + } + + private double getTwoDigitValue(double value) { + BigDecimal toBeTruncated = BigDecimal.valueOf(value); + return toBeTruncated.setScale(2, RoundingMode.HALF_UP).doubleValue(); + } + + private void adjustTemperature() { + setTemperature(); + Integer changedResource = adjustMinMaxMeasuredValue(currentTemp); + fireResourceChange(5700); + if (changedResource != null) { + fireResourceChange(changedResource); + } + } + + private void setTemperature(){ + float delta = (RANDOM.nextInt(20) - 10) / 10f; + currentTemp += delta; + } + private synchronized Integer adjustMinMaxMeasuredValue(double newTemperature) { + if (newTemperature > maxMeasuredValue) { + maxMeasuredValue = newTemperature; + return 5602; + } else if (newTemperature < minMeasuredValue) { + minMeasuredValue = newTemperature; + return 5601; + } else { + return null; + } + } + + private void resetMinMaxMeasuredValues() { + minMeasuredValue = currentTemp; + maxMeasuredValue = currentTemp; + } + + @Override + public List getAvailableResourceIds(ObjectModel model) { + return supportedResources; + } + + protected void setLeshanClient(LeshanClient leshanClient){ + this.leshanClient = leshanClient; + } + + @Override + public void destroy() { + } + + private void sendCollected(int resourceId) { + try { + LwM2mServer registeredServer = this.leshanClient.getRegisteredServers().values().iterator().next(); + ManualDataSender sender = this.leshanClient.getSendService().getDataSender(ManualDataSender.DEFAULT_NAME, + ManualDataSender.class); + sender.collectData(Arrays.asList(getPathForCollectedValue(resourceId))); + Thread.sleep(1000); + sender.collectData(Arrays.asList(getPathForCollectedValue(resourceId))); + sender.sendCollectedData(registeredServer, ContentFormat.SENML_JSON, 1000, false); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private LwM2mPath getPathForCollectedValue(int resourceId) { + return new LwM2mPath(3303, this.getId(), resourceId); + } + + private double getData() { + if (containingValues.size() > 1) { + Integer t0 = Math.toIntExact(Math.round(containingValues.get(0) * 100)); + Integer t1 = Math.toIntExact(Math.round(containingValues.get(1) * 100)); + long to_t1 = (((long) t0) << 32) | (t1 & 0xffffffffL); + return Double.longBitsToDouble(to_t1); + } else { + return currentTemp; + } + + } + + private void setData() { + if (containingValues == null){ + containingValues = new ArrayList<>(); + } + containingValues.add(getTwoDigitValue(currentTemp)); + } +} + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/LwM2mValueConverterImpl.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mValueConverterImpl.java similarity index 99% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/LwM2mValueConverterImpl.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mValueConverterImpl.java index b49d54f27e..bb76569685 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/LwM2mValueConverterImpl.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2mValueConverterImpl.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa.connectivity.lwm2m; +package org.thingsboard.server.msa.connectivity.lwm2m.client; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.model.ResourceModel.Type; diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mTestHelper.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/Lwm2mTestHelper.java similarity index 90% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mTestHelper.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/Lwm2mTestHelper.java index dfb4c71a1e..059876e94b 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mTestHelper.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/Lwm2mTestHelper.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa.connectivity.lwm2m; +package org.thingsboard.server.msa.connectivity.lwm2m.client; import org.eclipse.californium.elements.config.Configuration; import org.eclipse.leshan.client.object.Security; @@ -44,16 +44,22 @@ public class Lwm2mTestHelper { public static final String CLIENT_PSK_IDENTITY = "SOME_PSK_ID"; public static final String CLIENT_PSK_KEY = "73656372657450534b73656372657450"; + public static String OBSERVE_ATTRIBUTES_WITH_PARAMS = - public static final String OBSERVE_ATTRIBUTES_WITHOUT_PARAMS = " {\n" + - " \"keyName\": {},\n" + - " \"observe\": [],\n" + - " \"attribute\": [],\n" + - " \"telemetry\": [],\n" + + " \"keyName\": {\n" + + " \"/3_1.2/0/9\": \"batteryLevel\"\n" + + " },\n" + + " \"observe\": [\n" + + " \"/3_1.2/0/9\"\n" + + " ],\n" + + " \"attribute\": [\n" + + " ],\n" + + " \"telemetry\": [\n" + + " \"/3_1.2/0/9\"\n" + + " ],\n" + " \"attributeLwm2m\": {}\n" + " }"; - public static final String CLIENT_LWM2M_SETTINGS = " {\n" + " \"edrxCycle\": null,\n" + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/SimpleLwM2MDevice.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/SimpleLwM2MDevice.java similarity index 79% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/SimpleLwM2MDevice.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/SimpleLwM2MDevice.java index d2aa1af4d2..eefeb4bdf8 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/SimpleLwM2MDevice.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/SimpleLwM2MDevice.java @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa.connectivity.lwm2m; +package org.thingsboard.server.msa.connectivity.lwm2m.client; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.client.resource.BaseInstanceEnabler; import org.eclipse.leshan.client.servers.LwM2mServer; @@ -36,8 +37,11 @@ import java.util.Map; import java.util.PrimitiveIterator; import java.util.Random; import java.util.TimeZone; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; @Slf4j +@Data public class SimpleLwM2MDevice extends BaseInstanceEnabler implements Destroyable { @@ -46,17 +50,30 @@ public class SimpleLwM2MDevice extends BaseInstanceEnabler implements Destroyabl private static final int max = 50; private static final PrimitiveIterator.OfInt randomIterator = new Random().ints(min,max + 1).iterator(); private static final List supportedResources = Arrays.asList(0, 1, 2, 3, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21); - + private int countReadObserveAfterUpdateRegistrationSuccess_3_0_9; + private int countUpdateRegistrationSuccessLast; + private LwM2MTestClient lwM2MTestClient; public SimpleLwM2MDevice() { } - + public SimpleLwM2MDevice(ScheduledExecutorService executorService) { + try { + executorService.scheduleWithFixedDelay(() -> { + fireResourceChange(9); + } + , 1, 1, TimeUnit.SECONDS); // 30 MIN +// , 1800000, 1800000, TimeUnit.MILLISECONDS); // 30 MIN + } catch (Throwable e) { + log.error("[{}]Throwable", e.toString()); + e.printStackTrace(); + } + } @Override public ReadResponse read(LwM2mServer identity, int resourceId) { if (!identity.isSystem()) - log.info("Read on Device resource /{}/{}/{}", getModel().id, getId(), resourceId); + log.trace("Read on Device resource /{}/{}/{}", getModel().id, getId(), resourceId); switch (resourceId) { case 0: return ReadResponse.success(resourceId, getManufacturer()); @@ -155,8 +172,15 @@ public class SimpleLwM2MDevice extends BaseInstanceEnabler implements Destroyabl } private int getBatteryLevel() { + int batteryLevel = randomIterator.nextInt(); + if (countUpdateRegistrationSuccessLast != this.lwM2MTestClient.getCountUpdateRegistrationSuccess()) { + countUpdateRegistrationSuccessLast = this.lwM2MTestClient.getCountUpdateRegistrationSuccess(); + countReadObserveAfterUpdateRegistrationSuccess_3_0_9 = 0; + } + countReadObserveAfterUpdateRegistrationSuccess_3_0_9++; + this.lwM2MTestClient.setCountReadObserveAfterUpdateRegistrationSuccess(countReadObserveAfterUpdateRegistrationSuccess_3_0_9); + log.info("Read on Device resource /{}/{}/9 batteryLevel = {}, cntReadAfterUpdateReg [{}] ", getModel().id, getId(), batteryLevel, countReadObserveAfterUpdateRegistrationSuccess_3_0_9); return randomIterator.nextInt(); -// return 42; } private long getMemoryFree() { @@ -212,6 +236,10 @@ public class SimpleLwM2MDevice extends BaseInstanceEnabler implements Destroyabl return supportedResources; } + protected void setLwM2MTestClient(LwM2MTestClient lwM2MTestClient){ + this.lwM2MTestClient = lwM2MTestClient; + } + @Override public void destroy() { } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java new file mode 100644 index 0000000000..3eab4560fc --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.connectivity.lwm2m.rpc; + +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.connectivity.lwm2m.AbstractLwm2mClientTest; +import org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mDevicesForTest; + +import static org.thingsboard.server.msa.ui.utils.Const.TENANT_EMAIL; +import static org.thingsboard.server.msa.ui.utils.Const.TENANT_PASSWORD; + +@DisableUIListeners +public class Lwm2mObserveCompositeTest extends AbstractLwm2mClientTest { + + private Lwm2mDevicesForTest lwm2mDevicesForTest; + + private final static String name = "lwm2m-NoSec-ObserveComposite"; + + @BeforeMethod + public void setUp() throws Exception { + testRestClient.login(TENANT_EMAIL, TENANT_PASSWORD); + this.lwm2mDevicesForTest = new Lwm2mDevicesForTest(initTest(name + "-profile" + RandomStringUtils.randomAlphanumeric(7))); + } + + @AfterMethod + public void tearDown() { + destroyAfter(this.lwm2mDevicesForTest); + } + + @Test + public void testObserveResource_Update_AfterUpdateRegistration() throws Exception { + createLwm2mDevicesForConnectNoSec( name + "-" + RandomStringUtils.randomAlphanumeric(7), this.lwm2mDevicesForTest ); + observeCompositeResource_Update_AfterUpdateRegistration_test(this.lwm2mDevicesForTest.getLwM2MTestClient(), this.lwm2mDevicesForTest.getLwM2MDeviceTest().getId().toString()); + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java new file mode 100644 index 0000000000..1e10a692c8 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.connectivity.lwm2m.rpc; + +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.server.msa.connectivity.lwm2m.AbstractLwm2mClientTest; +import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mDevicesForTest; + +import static org.thingsboard.server.msa.ui.utils.Const.TENANT_EMAIL; +import static org.thingsboard.server.msa.ui.utils.Const.TENANT_PASSWORD; + +@DisableUIListeners +public class Lwm2mObserveTest extends AbstractLwm2mClientTest { + + private Lwm2mDevicesForTest lwm2mDevicesForTest; + + private final static String name = "lwm2m-NoSec-Observe"; + + @BeforeMethod + public void setUp() throws Exception { + testRestClient.login(TENANT_EMAIL, TENANT_PASSWORD); + this.lwm2mDevicesForTest = new Lwm2mDevicesForTest(initTest(name + "-profile" + RandomStringUtils.randomAlphanumeric(7))); + } + + @AfterMethod + public void tearDown() { + destroyAfter(this.lwm2mDevicesForTest); + } + + @Test + public void testObserveResource_Update_AfterUpdateRegistration() throws Exception { + createLwm2mDevicesForConnectNoSec( name + "-" + RandomStringUtils.randomAlphanumeric(7), this.lwm2mDevicesForTest ); + observeResource_Update_AfterUpdateRegistration_test(this.lwm2mDevicesForTest.getLwM2MTestClient(), this.lwm2mDevicesForTest.getLwM2MDeviceTest().getId().toString()); + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/Lwm2mClientNoSecTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/security/Lwm2mClientNoSecTest.java similarity index 60% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/Lwm2mClientNoSecTest.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/security/Lwm2mClientNoSecTest.java index 15658140b0..c614dedf2f 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/Lwm2mClientNoSecTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/security/Lwm2mClientNoSecTest.java @@ -13,33 +13,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa.connectivity; +package org.thingsboard.server.msa.connectivity.lwm2m.security; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import org.thingsboard.server.msa.AbstractLwm2mClientTest; +import org.thingsboard.server.msa.connectivity.lwm2m.AbstractLwm2mClientTest; import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mDevicesForTest; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.CLIENT_ENDPOINT_NO_SEC; import static org.thingsboard.server.msa.ui.utils.Const.TENANT_EMAIL; import static org.thingsboard.server.msa.ui.utils.Const.TENANT_PASSWORD; @DisableUIListeners public class Lwm2mClientNoSecTest extends AbstractLwm2mClientTest { + private Lwm2mDevicesForTest lwm2mDevicesForTest; @BeforeMethod public void setUp() throws Exception { testRestClient.login(TENANT_EMAIL, TENANT_PASSWORD); - initTest("lwm2m-NoSec"); + this.lwm2mDevicesForTest = new Lwm2mDevicesForTest(initTest("lwm2m-NoSec-profile" + "-" + RandomStringUtils.randomAlphanumeric(7))); } @AfterMethod public void tearDown() { - destroyAfter(); + destroyAfter(this.lwm2mDevicesForTest); } @Test public void connectLwm2mClientNoSecWithLwm2mServer() throws Exception { - connectLwm2mClientNoSec(); + createLwm2mDevicesForConnectNoSec(CLIENT_ENDPOINT_NO_SEC, this.lwm2mDevicesForTest); + basicTestConnection(this.lwm2mDevicesForTest.getLwM2MTestClient(), "TestConnection Lwm2m NoSec (msa)"); } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/Lwm2mClientPskTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/security/Lwm2mClientPskTest.java similarity index 64% rename from msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/Lwm2mClientPskTest.java rename to msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/security/Lwm2mClientPskTest.java index fbd8139d8f..9eab67f999 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/Lwm2mClientPskTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/security/Lwm2mClientPskTest.java @@ -13,13 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.msa.connectivity; +package org.thingsboard.server.msa.connectivity.lwm2m.security; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import org.thingsboard.server.msa.AbstractLwm2mClientTest; +import org.thingsboard.server.msa.connectivity.lwm2m.AbstractLwm2mClientTest; import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.connectivity.lwm2m.Lwm2mDevicesForTest; import static org.thingsboard.server.msa.ui.utils.Const.TENANT_EMAIL; import static org.thingsboard.server.msa.ui.utils.Const.TENANT_PASSWORD; @@ -27,19 +29,21 @@ import static org.thingsboard.server.msa.ui.utils.Const.TENANT_PASSWORD; @DisableUIListeners public class Lwm2mClientPskTest extends AbstractLwm2mClientTest { + private Lwm2mDevicesForTest lwm2mDevicesForTest; @BeforeMethod public void setUp() throws Exception { testRestClient.login(TENANT_EMAIL, TENANT_PASSWORD); - initTest("lwm2m-Psk"); + this.lwm2mDevicesForTest = new Lwm2mDevicesForTest(initTest("lwm2m-Psk-profile" + "-" + RandomStringUtils.randomAlphanumeric(7))); } @AfterMethod public void tearDown() { - destroyAfter(); + destroyAfter(this.lwm2mDevicesForTest); } @Test public void connectLwm2mClientPskWithLwm2mServer() throws Exception { - connectLwm2mClientPsk(); + createLwm2mDevicesForConnectPsk(this.lwm2mDevicesForTest); + basicTestConnection(this.lwm2mDevicesForTest.getLwM2MTestClient(), "TestConnection Lwm2m Rpc (msa)"); } } diff --git a/msa/black-box-tests/src/test/resources/lwm2m-registry/19.xml b/msa/black-box-tests/src/test/resources/lwm2m-registry/19.xml new file mode 100644 index 0000000000..ffa21864b7 --- /dev/null +++ b/msa/black-box-tests/src/test/resources/lwm2m-registry/19.xml @@ -0,0 +1,144 @@ + + + + + BinaryAppDataContainer + + 19 + urn:oma:lwm2m:oma:19:1.1 + 1.1 + 1.1 + Multiple + Optional + + Data + RW + Multiple + Mandatory + Opaque + + + + + Data Priority + RW + Single + Optional + Integer + 1 bytes + + + + Data Creation Time + RW + Single + Optional + Time + + + + + Data Description + RW + Single + Optional + String + 32 bytes + + + + Data Format + RW + Single + Optional + String + 32 bytes + + + + App ID + RW + Single + Optional + Integer + 2 bytes + + + + + + diff --git a/msa/black-box-tests/src/test/resources/lwm2m-registry/3303.xml b/msa/black-box-tests/src/test/resources/lwm2m-registry/3303.xml new file mode 100644 index 0000000000..a6e5406b08 --- /dev/null +++ b/msa/black-box-tests/src/test/resources/lwm2m-registry/3303.xml @@ -0,0 +1,103 @@ + + + + + Temperature + This IPSO object should be used with a temperature sensor to report a temperature measurement. It also provides resources for minimum/maximum measured values and the minimum/maximum range that can be measured by the temperature sensor. An example measurement unit is degrees Celsius. + 3303 + urn:oma:lwm2m:ext:3303 + 1.0 + 1.0 + Multiple + Optional + + + Sensor Value + R + Single + Mandatory + Float + + + Last or Current Measured Value from the Sensor + + + Min Measured Value + R + Single + Optional + Float + + + The minimum value measured by the sensor since power ON or reset + + + Max Measured Value + R + Single + Optional + Float + + + The maximum value measured by the sensor since power ON or reset + + + Min Range Value + R + Single + Optional + Float + + + The minimum value that can be measured by the sensor + + + Max Range Value + R + Single + Optional + Float + + + The maximum value that can be measured by the sensor + + + Sensor Units + R + Single + Optional + String + + + Measurement Units Definition. + + + Reset Min and Max Measured Values + E + Single + Optional + + + + Reset the Min and Max Measured Values to Current Value + + + + + From c548465a2c066ddb86d467c46382a783fb60cc5c Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 9 Sep 2024 09:04:39 +0300 Subject: [PATCH 07/32] lwm2m: fix bug Observe Composite - license --- .../connectivity/lwm2m/Lwm2mDevicesForTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java index 5374449e11..b1e85c0e38 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/Lwm2mDevicesForTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.msa.connectivity.lwm2m; From d4c4bf56562b4a6dad43a02cc0b68654d95bb6ba Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 9 Sep 2024 22:11:22 +0200 Subject: [PATCH 08/32] implemented new API for gateway latency --- .../src/main/resources/thingsboard.yml | 2 + ...AbstractMqttTimeseriesIntegrationTest.java | 98 ++++++++++++++ .../data/device/profile/MqttTopics.java | 1 + .../transport/mqtt/MqttTransportContext.java | 8 +- .../transport/mqtt/MqttTransportHandler.java | 10 +- .../transport/mqtt/MqttTransportService.java | 2 +- .../mqtt/TbMqttTransportComponent.java | 26 ++++ .../mqtt/gateway/GatewayLatencyService.java | 123 ++++++++++++++++++ .../gateway/latency/GatewayLatencyData.java | 19 +++ .../gateway/latency/GatewayLatencyState.java | 119 +++++++++++++++++ .../mqtt/session/GatewaySessionHandler.java | 49 ++++++- 11 files changed, 451 insertions(+), 6 deletions(-) create mode 100644 common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttTransportComponent.java create mode 100644 common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java create mode 100644 common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java create mode 100644 common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 785f124583..56dfe0ab46 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1020,6 +1020,8 @@ transport: # MQTT disconnect timeout in milliseconds. The time to wait for the client to disconnect after the server sends a disconnect message. disconnect_timeout: "${MQTT_DISCONNECT_TIMEOUT:1000}" msg_queue_size_per_device_limit: "${MQTT_MSG_QUEUE_SIZE_PER_DEVICE_LIMIT:100}" # messages await in the queue before the device connected state. This limit works on the low level before TenantProfileLimits mechanism + # Interval of periodic report of the gateway latency + gateway_latency_report_interval_sec: "${MQTT_GATEWAY_LATENCY_REPORT_INTERVAL_SEC:3600}" netty: # Netty leak detector level leak_detector_level: "${NETTY_LEAK_DETECTOR_LVL:DISABLED}" diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index 7c8cdb5d6e..6deceb6fb9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -18,30 +18,45 @@ package org.thingsboard.server.transport.mqtt.mqttv3.telemetry.timeseries; import com.fasterxml.jackson.core.type.TypeReference; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest; import org.thingsboard.server.transport.mqtt.MqttTestConfigProperties; +import org.thingsboard.server.transport.mqtt.gateway.GatewayLatencyService; +import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyData; +import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyState; import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestCallback; import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestClient; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.OptionalLong; +import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_ATTRIBUTES_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_TELEMETRY_SHORT_JSON_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_TELEMETRY_SHORT_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_TELEMETRY_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_CONNECT_TOPIC; +import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_LATENCY_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_TELEMETRY_TOPIC; @Slf4j @@ -53,6 +68,9 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt protected static final String MALFORMED_JSON_PAYLOAD = "{\"key1\":, \"key2\":true, \"key3\": 3.0, \"key4\": 4," + " \"key5\": {\"someNumber\": 42, \"someArray\": [1,2,3], \"someNestedObject\": {\"key\": \"value\"}}}"; + @SpyBean + GatewayLatencyService gatewayLatencyService; + @Before public void beforeTest() throws Exception { MqttTestConfigProperties configProperties = MqttTestConfigProperties.builder() @@ -113,6 +131,86 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt client.disconnect(); } + @Test + public void testPushLatencyGateway() throws Exception { + MqttTestClient client = new MqttTestClient(); + client.connectAndWait(gatewayAccessToken); + + Map> gwLatencies = new HashMap<>(); + List transportLatencies = new ArrayList<>(); + + publishLatency(client, gwLatencies, transportLatencies, 5); + + gatewayLatencyService.reportLatency(); + + List actualKeys = getActualKeysList(savedGateway.getId(), List.of("latencyCheck")); + assertEquals("latencyCheck", actualKeys.get(0)); + + String telemetryUrl = String.format("/api/plugins/telemetry/DEVICE/%s/values/timeseries?startTs=%d&endTs=%d&keys=latencyCheck", savedGateway.getId(), 0, System.currentTimeMillis()); + + Map>> gatewayTelemetry = doGetAsyncTyped(telemetryUrl, new TypeReference<>() {}); + Map latencyCheckTelemetry = gatewayTelemetry.get("latencyCheck").get(0); + Map latencyCheckValue = JacksonUtil.fromString((String) latencyCheckTelemetry.get("value"), new TypeReference<>() {}); + assertNotNull(latencyCheckValue); + + long avgTransportLatency = (long) transportLatencies.stream().mapToLong(Long::longValue).average().getAsDouble(); + long minTransportLatency = transportLatencies.stream().mapToLong(Long::longValue).min().getAsLong(); + long maxTransportLatency = transportLatencies.stream().mapToLong(Long::longValue).max().getAsLong(); + + + gwLatencies.forEach((connectorName, gwLatencyList) -> { + long avgGwLatency = (long) gwLatencyList.stream().mapToLong(Long::longValue).average().getAsDouble(); + long minGwLatency = gwLatencyList.stream().mapToLong(Long::longValue).min().getAsLong(); + long maxGwLatency = gwLatencyList.stream().mapToLong(Long::longValue).max().getAsLong(); + + GatewayLatencyState.ConnectorLatencyResult connectorLatencyResult = latencyCheckValue.get(connectorName); + assertNotNull(connectorLatencyResult); + checkConnectorLatencyResult(connectorLatencyResult, avgGwLatency, minGwLatency, maxGwLatency, avgTransportLatency, minTransportLatency, maxTransportLatency); + }); + + client.disconnect(); + + Awaitility.await() + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> verify(gatewayLatencyService).onDeviceDisconnect(savedGateway.getId())); + } + + private void publishLatency(MqttTestClient client, Map> gwLatencies, List transportLatencies, int n) throws Exception { + Random random = new Random(); + for (int i = 0; i < n; i++) { + long publishedTs = System.currentTimeMillis(); + long gatewayLatencyA = random.nextLong(1000, 5000); + long gatewayLatencyB = random.nextLong(1200, 4500); + long transportReceiveTs = publishLatencyAndGetTransportReceiveTs(client, publishedTs, gatewayLatencyA, gatewayLatencyB); + gwLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(gatewayLatencyA); + gwLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(gatewayLatencyB); + transportLatencies.add(transportReceiveTs - publishedTs); + Thread.sleep(1); + } + } + + private long publishLatencyAndGetTransportReceiveTs(MqttTestClient client, long publishedTs, long gatewayLatencyA, long gatewayLatencyB) throws Exception { + List data = new ArrayList<>(); + data.add(new GatewayLatencyData("connectorA", publishedTs - gatewayLatencyA, publishedTs)); + data.add(new GatewayLatencyData("connectorB", publishedTs - gatewayLatencyB, publishedTs)); + + client.publishAndWait(GATEWAY_LATENCY_TOPIC, JacksonUtil.writeValueAsBytes(data)); + ArgumentCaptor transportReceiveTsCaptor = ArgumentCaptor.forClass(Long.class); + verify(gatewayLatencyService).process(any(), eq(savedGateway.getId()), eq(data), transportReceiveTsCaptor.capture()); + return transportReceiveTsCaptor.getValue(); + } + + private void checkConnectorLatencyResult(GatewayLatencyState.ConnectorLatencyResult result, long avgGwLatency, long minGwLatency, long maxGwLatency, + long avgTransportLatency, long minTransportLatency, long maxTransportLatency) { + assertNotNull(result); + assertEquals(avgGwLatency, result.avgGwLatency()); + assertEquals(minGwLatency, result.minGwLatency()); + assertEquals(maxGwLatency, result.maxGwLatency()); + assertEquals(avgTransportLatency, result.transportLatencyAvg()); + assertEquals(minTransportLatency, result.minTransportLatency()); + assertEquals(maxTransportLatency, result.maxTransportLatency()); + } + protected void processJsonPayloadTelemetryTest(String topic, List expectedKeys, byte[] payload, boolean withTs) throws Exception { processTelemetryTest(topic, expectedKeys, payload, withTs, false); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java index cabdea145c..8945f2d084 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java @@ -75,6 +75,7 @@ public class MqttTopics { public static final String GATEWAY_RPC_TOPIC = BASE_GATEWAY_API_TOPIC + RPC; public static final String GATEWAY_ATTRIBUTES_REQUEST_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_REQUEST; public static final String GATEWAY_ATTRIBUTES_RESPONSE_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_RESPONSE; + public static final String GATEWAY_LATENCY_TOPIC = BASE_GATEWAY_API_TOPIC + "/latency"; // v2 topics public static final String BASE_DEVICE_API_TOPIC_V2 = "v2"; public static final String REQUEST_ID_PATTERN = "(?\\d+)"; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 16d72ff89f..555d7de868 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -22,11 +22,11 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.transport.mqtt.adaptors.JsonMqttAdaptor; import org.thingsboard.server.transport.mqtt.adaptors.ProtoMqttAdaptor; +import org.thingsboard.server.transport.mqtt.gateway.GatewayLatencyService; import java.net.InetSocketAddress; import java.util.concurrent.atomic.AtomicInteger; @@ -36,7 +36,7 @@ import java.util.concurrent.atomic.AtomicInteger; */ @Slf4j @Component -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')") +@TbMqttTransportComponent public class MqttTransportContext extends TransportContext { @Getter @@ -51,6 +51,10 @@ public class MqttTransportContext extends TransportContext { @Autowired private ProtoMqttAdaptor protoMqttAdaptor; + @Getter + @Autowired + private GatewayLatencyService gatewayLatencyService; + @Getter @Value("${transport.mqtt.netty.max_payload_size}") private Integer maxPayloadSize; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index eeeb287dc8..ef63fb5902 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -419,6 +419,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_DISCONNECT_TOPIC: gatewaySessionHandler.onDeviceDisconnect(mqttMsg); break; + case MqttTopics.GATEWAY_LATENCY_TOPIC: + gatewaySessionHandler.onGatewayLatency(mqttMsg); + break; default: ack(ctx, msgId, MqttReasonCodes.PubAck.TOPIC_NAME_INVALID); } @@ -1187,7 +1190,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement transportService.process(deviceSessionCtx.getSessionInfo(), SESSION_EVENT_MSG_CLOSED, null); transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); if (gatewaySessionHandler != null) { - gatewaySessionHandler.onDevicesDisconnect(); + gatewaySessionHandler.onGatewayDisconnect(); } if (sparkplugSessionHandler != null) { // add Msg Telemetry node: key STATE type: String value: OFFLINE ts: sparkplugBProto.getTimestamp() @@ -1418,7 +1421,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { deviceSessionCtx.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); if (gatewaySessionHandler != null) { - gatewaySessionHandler.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); + gatewaySessionHandler.onGatewayUpdate(sessionInfo, device, deviceProfileOpt); } } @@ -1427,6 +1430,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement context.onAuthFailure(address); ChannelHandlerContext ctx = deviceSessionCtx.getChannel(); closeCtx(ctx, MqttReasonCodes.Disconnect.ADMINISTRATIVE_ACTION); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayDelete(deviceId); + } } public void sendErrorRpcResponse(TransportProtos.SessionInfoProto sessionInfo, int requestId, ThingsboardErrorCode result, String errorMsg) { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index ffc7e9e561..b8d1dbe582 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -39,7 +39,7 @@ import java.net.InetSocketAddress; * @author Andrew Shvayka */ @Service("MqttTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')") +@TbMqttTransportComponent @Slf4j public class MqttTransportService implements TbTransportService { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttTransportComponent.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttTransportComponent.java new file mode 100644 index 0000000000..da0047cc7e --- /dev/null +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/TbMqttTransportComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.mqtt; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')") +public @interface TbMqttTransportComponent { +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java new file mode 100644 index 0000000000..c6e9002877 --- /dev/null +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.mqtt.gateway; + +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; +import org.thingsboard.server.transport.mqtt.TbMqttTransportComponent; +import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyData; +import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyState; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Service +@TbMqttTransportComponent +public class GatewayLatencyService { + + @Value("${transport.mqtt.gateway_latency_report_interval_sec:3600}") + private int latencyReportIntervalSec; + + @Autowired + private SchedulerComponent scheduler; + + @Autowired + private TransportService transportService; + + private Map states = new ConcurrentHashMap<>(); + + @PostConstruct + private void init() { + scheduler.scheduleAtFixedRate(this::reportLatency, latencyReportIntervalSec, latencyReportIntervalSec, TimeUnit.SECONDS); + } + + public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, List data, long ts) { + states.computeIfAbsent(gatewayId, k -> new GatewayLatencyState(sessionInfo)).update(ts, data); + } + + public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId) { + var state = states.get(gatewayId); + if (state != null) { + state.updateSessionInfo(sessionInfo); + } + } + + public void onDeviceDelete(DeviceId deviceId) { + var state = states.remove(deviceId); + if (state != null) { + state.clear(); + } + } + + public void onDeviceDisconnect(DeviceId deviceId) { + GatewayLatencyState state = states.remove(deviceId); + if (state != null) { + reportLatency(state, System.currentTimeMillis()); + } + } + + public void reportLatency() { + if (states.isEmpty()) { + return; + } + Map oldStates = states; + states = new ConcurrentHashMap<>(); + + long ts = System.currentTimeMillis(); + + oldStates.forEach((gatewayId, state) -> { + reportLatency(state, ts); + }); + oldStates.clear(); + } + + private void reportLatency(GatewayLatencyState state, long ts) { + if (state.isEmpty()) { + return; + } + var result = state.getLatencyStateResult(); + state.clear(); + var kvProto = TransportProtos.KeyValueProto.newBuilder() + .setKey("latencyCheck") + .setType(TransportProtos.KeyValueType.JSON_V) + .setJsonV(JacksonUtil.toString(result)) + .build(); + + TransportProtos.TsKvListProto tsKvList = TransportProtos.TsKvListProto.newBuilder() + .setTs(ts) + .addKv(kvProto) + .build(); + + TransportProtos.PostTelemetryMsg telemetryMsg = TransportProtos.PostTelemetryMsg.newBuilder() + .addTsKvList(tsKvList) + .build(); + + transportService.process(state.getSessionInfo(), telemetryMsg, TransportServiceCallback.EMPTY); + } + +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java new file mode 100644 index 0000000000..6260c8356e --- /dev/null +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.mqtt.gateway.latency; + +public record GatewayLatencyData(String connectorName, long receivedTs, long publishedTs) { +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java new file mode 100644 index 0000000000..3ed902f81d --- /dev/null +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.mqtt.gateway.latency; + +import lombok.Getter; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public class GatewayLatencyState { + + private final Map connectors; + private final Lock updateLock; + + @Getter + private volatile TransportProtos.SessionInfoProto sessionInfo; + + public GatewayLatencyState(TransportProtos.SessionInfoProto sessionInfo) { + this.connectors = new HashMap<>(); + this.updateLock = new ReentrantLock(); + this.sessionInfo = sessionInfo; + } + + public void updateSessionInfo(TransportProtos.SessionInfoProto sessionInfo) { + this.sessionInfo = sessionInfo; + } + + public void update(long ts, List latencyData) { + updateLock.lock(); + try { + latencyData.forEach(data -> { + connectors.computeIfAbsent(data.connectorName(), k -> new ConnectorLatencyState()).update(ts, data); + }); + } finally { + updateLock.unlock(); + } + } + + public void clear() { + connectors.clear(); + } + + public Map getLatencyStateResult() { + Map result = new HashMap<>(); + connectors.forEach((name, state) -> result.put(name, state.getResult())); + return result; + } + + public boolean isEmpty() { + return connectors.isEmpty(); + } + + private static class ConnectorLatencyState { + private final AtomicInteger count; + private final AtomicLong gwLatencySum; + private final AtomicLong transportLatencySum; + private volatile long minGwLatency; + private volatile long maxGwLatency; + private volatile long minTransportLatency; + private volatile long maxTransportLatency; + + private ConnectorLatencyState() { + this.count = new AtomicInteger(0); + this.gwLatencySum = new AtomicLong(0); + this.transportLatencySum = new AtomicLong(0); + } + + private void update(long serverReceiveTs, GatewayLatencyData latencyData) { + long gwLatency = latencyData.publishedTs() - latencyData.receivedTs(); + long transportLatency = serverReceiveTs - latencyData.publishedTs(); + count.incrementAndGet(); + gwLatencySum.addAndGet(gwLatency); + transportLatencySum.addAndGet(transportLatency); + if (minGwLatency == 0 || minGwLatency > gwLatency) { + minGwLatency = gwLatency; + } + if (maxGwLatency < gwLatency) { + maxGwLatency = gwLatency; + } + if (minTransportLatency == 0 || minTransportLatency > transportLatency) { + minTransportLatency = transportLatency; + } + if (maxTransportLatency < transportLatency) { + maxTransportLatency = transportLatency; + } + } + + private ConnectorLatencyResult getResult() { + long count = this.count.get(); + long avgGwLatency = gwLatencySum.get() / count; + long transportLatencyAvg = transportLatencySum.get() / count; + return new ConnectorLatencyResult(avgGwLatency, minGwLatency, maxGwLatency, transportLatencyAvg, minTransportLatency, maxTransportLatency); + } + } + + public record ConnectorLatencyResult(long avgGwLatency, long minGwLatency, long maxGwLatency, + long transportLatencyAvg, long minTransportLatency, long maxTransportLatency) { + } + +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index 9aa228afff..684f6b3114 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -15,20 +15,36 @@ */ package org.thingsboard.server.transport.mqtt.session; +import com.fasterxml.jackson.core.type.TypeReference; import io.netty.buffer.ByteBuf; import io.netty.handler.codec.mqtt.MqttPublishMessage; +import io.netty.handler.codec.mqtt.MqttReasonCodes; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.transport.mqtt.gateway.GatewayLatencyService; +import java.util.Optional; import java.util.UUID; +import static com.amazonaws.util.StringUtils.UTF8; + /** * Created by nickAS21 on 26.12.22 */ +@Slf4j public class GatewaySessionHandler extends AbstractGatewaySessionHandler { + private final GatewayLatencyService latencyService; + public GatewaySessionHandler(DeviceSessionCtx deviceSessionCtx, UUID sessionId, boolean overwriteDevicesActivity) { super(deviceSessionCtx, sessionId, overwriteDevicesActivity); + this.latencyService = deviceSessionCtx.getContext().getGatewayLatencyService(); } public void onDeviceConnect(MqttPublishMessage mqttMsg) throws AdaptorException { @@ -51,7 +67,38 @@ public class GatewaySessionHandler extends AbstractGatewaySessionHandler deviceProfileOpt) { + this.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); + latencyService.onDeviceUpdate(sessionInfo, gateway.getDeviceId()); + } + + public void onGatewayDelete(DeviceId deviceId) { + latencyService.onDeviceDelete(deviceId); + } + + public void onGatewayLatency(MqttPublishMessage mqttMsg) throws AdaptorException { + int msgId = getMsgId(mqttMsg); + ByteBuf payloadData = mqttMsg.payload(); + String payload = payloadData.toString(UTF8); + if (payload == null) { + log.debug("[{}][{}][{}] Payload is empty!", gateway.getTenantId(), gateway.getDeviceId(), sessionId); + throw new AdaptorException(new IllegalArgumentException("Payload is empty!")); + } + long ts = System.currentTimeMillis(); + try { + latencyService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), JacksonUtil.fromString(payload, new TypeReference<>() {}), ts); + ack(msgId, MqttReasonCodes.PubAck.SUCCESS); + } catch (IllegalArgumentException e) { + throw new AdaptorException(e); + } } } From 06f30e1de534b6946f4dc08206024dca5de2e9e0 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 9 Sep 2024 23:00:29 +0200 Subject: [PATCH 09/32] minor refactoring --- ...AbstractMqttTimeseriesIntegrationTest.java | 55 ++++++++++--------- .../mqtt/gateway/GatewayLatencyService.java | 3 +- .../gateway/latency/GatewayLatencyData.java | 2 +- .../gateway/latency/GatewayLatencyState.java | 7 +-- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index 6deceb6fb9..2aeecfb663 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -40,7 +40,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.OptionalLong; import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -137,7 +136,7 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt client.connectAndWait(gatewayAccessToken); Map> gwLatencies = new HashMap<>(); - List transportLatencies = new ArrayList<>(); + Map> transportLatencies = new HashMap<>(); publishLatency(client, gwLatencies, transportLatencies, 5); @@ -153,16 +152,18 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt Map latencyCheckValue = JacksonUtil.fromString((String) latencyCheckTelemetry.get("value"), new TypeReference<>() {}); assertNotNull(latencyCheckValue); - long avgTransportLatency = (long) transportLatencies.stream().mapToLong(Long::longValue).average().getAsDouble(); - long minTransportLatency = transportLatencies.stream().mapToLong(Long::longValue).min().getAsLong(); - long maxTransportLatency = transportLatencies.stream().mapToLong(Long::longValue).max().getAsLong(); - - gwLatencies.forEach((connectorName, gwLatencyList) -> { long avgGwLatency = (long) gwLatencyList.stream().mapToLong(Long::longValue).average().getAsDouble(); long minGwLatency = gwLatencyList.stream().mapToLong(Long::longValue).min().getAsLong(); long maxGwLatency = gwLatencyList.stream().mapToLong(Long::longValue).max().getAsLong(); + List transportLatencyList = transportLatencies.get(connectorName); + assertNotNull(transportLatencyList); + + long avgTransportLatency = (long) transportLatencyList.stream().mapToLong(Long::longValue).average().getAsDouble(); + long minTransportLatency = transportLatencyList.stream().mapToLong(Long::longValue).min().getAsLong(); + long maxTransportLatency = transportLatencyList.stream().mapToLong(Long::longValue).max().getAsLong(); + GatewayLatencyState.ConnectorLatencyResult connectorLatencyResult = latencyCheckValue.get(connectorName); assertNotNull(connectorLatencyResult); checkConnectorLatencyResult(connectorLatencyResult, avgGwLatency, minGwLatency, maxGwLatency, avgTransportLatency, minTransportLatency, maxTransportLatency); @@ -175,29 +176,31 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt .untilAsserted(() -> verify(gatewayLatencyService).onDeviceDisconnect(savedGateway.getId())); } - private void publishLatency(MqttTestClient client, Map> gwLatencies, List transportLatencies, int n) throws Exception { + private void publishLatency(MqttTestClient client, Map> gwLatencies, Map> transportLatencies, int n) throws Exception { Random random = new Random(); for (int i = 0; i < n; i++) { - long publishedTs = System.currentTimeMillis(); - long gatewayLatencyA = random.nextLong(1000, 5000); - long gatewayLatencyB = random.nextLong(1200, 4500); - long transportReceiveTs = publishLatencyAndGetTransportReceiveTs(client, publishedTs, gatewayLatencyA, gatewayLatencyB); + Map data = new HashMap<>(); + long publishedTs = System.currentTimeMillis() - 10; + long gatewayLatencyA = random.nextLong(100, 500); + data.put("connectorA", new GatewayLatencyData(publishedTs - gatewayLatencyA, publishedTs)); gwLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(gatewayLatencyA); - gwLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(gatewayLatencyB); - transportLatencies.add(transportReceiveTs - publishedTs); - Thread.sleep(1); - } - } - - private long publishLatencyAndGetTransportReceiveTs(MqttTestClient client, long publishedTs, long gatewayLatencyA, long gatewayLatencyB) throws Exception { - List data = new ArrayList<>(); - data.add(new GatewayLatencyData("connectorA", publishedTs - gatewayLatencyA, publishedTs)); - data.add(new GatewayLatencyData("connectorB", publishedTs - gatewayLatencyB, publishedTs)); + boolean sendB = i % 2 == 0; + if (sendB) { + long gatewayLatencyB = random.nextLong(120, 450); + data.put("connectorB", new GatewayLatencyData(publishedTs - gatewayLatencyB, publishedTs)); + gwLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(gatewayLatencyB); + } - client.publishAndWait(GATEWAY_LATENCY_TOPIC, JacksonUtil.writeValueAsBytes(data)); - ArgumentCaptor transportReceiveTsCaptor = ArgumentCaptor.forClass(Long.class); - verify(gatewayLatencyService).process(any(), eq(savedGateway.getId()), eq(data), transportReceiveTsCaptor.capture()); - return transportReceiveTsCaptor.getValue(); + client.publishAndWait(GATEWAY_LATENCY_TOPIC, JacksonUtil.writeValueAsBytes(data)); + ArgumentCaptor transportReceiveTsCaptor = ArgumentCaptor.forClass(Long.class); + verify(gatewayLatencyService).process(any(), eq(savedGateway.getId()), eq(data), transportReceiveTsCaptor.capture()); + Long transportReceiveTs = transportReceiveTsCaptor.getValue(); + Long transportLatency = transportReceiveTs - publishedTs; + transportLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(transportLatency); + if (sendB) { + transportLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(transportLatency); + } + } } private void checkConnectorLatencyResult(GatewayLatencyState.ConnectorLatencyResult result, long avgGwLatency, long minGwLatency, long maxGwLatency, diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java index c6e9002877..148a8819ee 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java @@ -30,7 +30,6 @@ import org.thingsboard.server.transport.mqtt.TbMqttTransportComponent; import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyData; import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyState; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -56,7 +55,7 @@ public class GatewayLatencyService { scheduler.scheduleAtFixedRate(this::reportLatency, latencyReportIntervalSec, latencyReportIntervalSec, TimeUnit.SECONDS); } - public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, List data, long ts) { + public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, Map data, long ts) { states.computeIfAbsent(gatewayId, k -> new GatewayLatencyState(sessionInfo)).update(ts, data); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java index 6260c8356e..2cae428a02 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java @@ -15,5 +15,5 @@ */ package org.thingsboard.server.transport.mqtt.gateway.latency; -public record GatewayLatencyData(String connectorName, long receivedTs, long publishedTs) { +public record GatewayLatencyData(long receivedTs, long publishedTs) { } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java index 3ed902f81d..c139bb3f19 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java @@ -19,7 +19,6 @@ import lombok.Getter; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -44,11 +43,11 @@ public class GatewayLatencyState { this.sessionInfo = sessionInfo; } - public void update(long ts, List latencyData) { + public void update(long ts, Map latencyData) { updateLock.lock(); try { - latencyData.forEach(data -> { - connectors.computeIfAbsent(data.connectorName(), k -> new ConnectorLatencyState()).update(ts, data); + latencyData.forEach((connectorName, data) -> { + connectors.computeIfAbsent(connectorName, k -> new ConnectorLatencyState()).update(ts, data); }); } finally { updateLock.unlock(); From db88ed1aab4c4d300830ad26dcaeea632b5d59eb Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 11 Sep 2024 01:08:41 +0900 Subject: [PATCH 10/32] chore: update EdgeGrpcSession.java attemps -> attempts --- .../thingsboard/server/service/edge/rpc/EdgeGrpcSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index c397807c5f..d0d10f0174 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -110,7 +110,7 @@ public final class EdgeGrpcSession implements Closeable { private static final ReentrantLock downlinkMsgLock = new ReentrantLock(); private static final ConcurrentLinkedQueue highPriorityQueue = new ConcurrentLinkedQueue<>(); - private static final int MAX_DOWNLINK_ATTEMPTS = 10; // max number of attemps to send downlink message if edge connected + private static final int MAX_DOWNLINK_ATTEMPTS = 10; // max number of attempts to send downlink message if edge connected private static final String QUEUE_START_TS_ATTR_KEY = "queueStartTs"; private static final String QUEUE_START_SEQ_ID_ATTR_KEY = "queueStartSeqId"; From fe571beaa9864c29443c8111d4eaf347300e5a57 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 11 Sep 2024 09:50:00 +0300 Subject: [PATCH 11/32] fixed minor comments in tests --- .../engine/metadata/TbGetTelemetryNodeTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java index c779afe4c5..f2d4a41fbc 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java @@ -208,7 +208,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, DEVICE_ID, metaData, "{\"msgEndInterval\":\"" + endTs + "\"}"); node.onMsg(ctxMock, msg); - /// THEN + // THEN ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); @@ -258,12 +258,12 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); List actualKeys = actualReadTsKvQueryList.getValue().stream().map(TsKvQuery::getKey).toList(); - assertThat(actualKeys).containsAll(List.of("temperature", "humidity", "pressure")); + assertThat(actualKeys).containsExactlyInAnyOrder("temperature", "humidity", "pressure"); } @ParameterizedTest @MethodSource - public void givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery(Aggregation aggregation, Consumer verifyAggregationStepInQuery) throws TbNodeException { + public void givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery(Aggregation aggregation, Consumer aggregationStepVerifier) throws TbNodeException { // GIVEN config.setStartInterval(5); config.setEndInterval(1); @@ -283,7 +283,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); - verifyAggregationStepInQuery.accept(actualReadTsKvQuery); + aggregationStepVerifier.accept(actualReadTsKvQuery); } private static Stream givenAggregation_whenOnMsg_thenVerifyAggregationStepInQuery() { @@ -295,7 +295,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { @ParameterizedTest @MethodSource - public void givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery(FetchMode fetchMode, int limit, Consumer verifyLimitInQuery) throws TbNodeException { + public void givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery(FetchMode fetchMode, int limit, Consumer limitInQueryVerifier) throws TbNodeException { // GIVEN config.setFetchMode(fetchMode); config.setLimit(limit); @@ -313,7 +313,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); - verifyLimitInQuery.accept(actualReadTsKvQuery); + limitInQueryVerifier.accept(actualReadTsKvQuery); } private static Stream givenFetchModeAndLimit_whenOnMsg_thenVerifyLimitInQuery() { @@ -335,7 +335,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { @ParameterizedTest @MethodSource - public void givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery(FetchMode fetchMode, Direction orderBy, Consumer verifyOrderInQuery) throws TbNodeException { + public void givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery(FetchMode fetchMode, Direction orderBy, Consumer orderInQueryVerifier) throws TbNodeException { // GIVEN config.setFetchMode(fetchMode); config.setOrderBy(orderBy); @@ -353,7 +353,7 @@ public class TbGetTelemetryNodeTest extends AbstractRuleNodeUpgradeTest { ArgumentCaptor> actualReadTsKvQueryList = ArgumentCaptor.forClass(List.class); then(timeseriesServiceMock).should().findAll(eq(TENANT_ID), eq(DEVICE_ID), actualReadTsKvQueryList.capture()); ReadTsKvQuery actualReadTsKvQuery = actualReadTsKvQueryList.getValue().get(0); - verifyOrderInQuery.accept(actualReadTsKvQuery); + orderInQueryVerifier.accept(actualReadTsKvQuery); } private static Stream givenFetchModeAndOrder_whenOnMsg_thenVerifyOrderInQuery() { From f64ec5ca56f9c884936bd22a40969d539394594c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 11 Sep 2024 10:25:24 +0200 Subject: [PATCH 12/32] test fixes --- .../timeseries/AbstractMqttTimeseriesIntegrationTest.java | 7 +++++++ .../transport/mqtt/session/GatewaySessionHandler.java | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index 2aeecfb663..886cf1e309 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -25,6 +25,7 @@ import org.mockito.ArgumentCaptor; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest; import org.thingsboard.server.transport.mqtt.MqttTestConfigProperties; @@ -57,6 +58,7 @@ import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVIC import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_CONNECT_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_LATENCY_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_TELEMETRY_TOPIC; +import static org.thingsboard.server.transport.mqtt.mqttv3.credentials.BasicMqttCredentialsTest.CLIENT_ID; @Slf4j public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqttIntegrationTest { @@ -132,6 +134,11 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt @Test public void testPushLatencyGateway() throws Exception { + MqttTestConfigProperties configProperties = MqttTestConfigProperties.builder() + .gatewayName("Test latency gateway") + .build(); + processBeforeTest(configProperties); + MqttTestClient client = new MqttTestClient(); client.connectAndWait(gatewayAccessToken); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index 684f6b3114..a7634593ce 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -96,8 +96,8 @@ public class GatewaySessionHandler extends AbstractGatewaySessionHandler() {}), ts); ack(msgId, MqttReasonCodes.PubAck.SUCCESS); - } catch (IllegalArgumentException e) { - throw new AdaptorException(e); + } catch (Throwable t) { + ackOrClose(msgId); } } From dfb633b8e8a31a19b890c3158ed2067f172f0458 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 11 Sep 2024 11:26:04 +0200 Subject: [PATCH 13/32] improvements --- .../mqtt/gateway/GatewayLatencyService.java | 6 +----- .../mqtt/gateway/latency/GatewayLatencyState.java | 13 ++++++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java index 148a8819ee..acf10a03b8 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java @@ -67,10 +67,7 @@ public class GatewayLatencyService { } public void onDeviceDelete(DeviceId deviceId) { - var state = states.remove(deviceId); - if (state != null) { - state.clear(); - } + states.remove(deviceId); } public void onDeviceDisconnect(DeviceId deviceId) { @@ -100,7 +97,6 @@ public class GatewayLatencyService { return; } var result = state.getLatencyStateResult(); - state.clear(); var kvProto = TransportProtos.KeyValueProto.newBuilder() .setKey("latencyCheck") .setType(TransportProtos.KeyValueType.JSON_V) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java index c139bb3f19..3ad75fee75 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java @@ -54,13 +54,16 @@ public class GatewayLatencyState { } } - public void clear() { - connectors.clear(); - } - public Map getLatencyStateResult() { Map result = new HashMap<>(); - connectors.forEach((name, state) -> result.put(name, state.getResult())); + updateLock.lock(); + try { + connectors.forEach((name, state) -> result.put(name, state.getResult())); + connectors.clear(); + } finally { + updateLock.unlock(); + } + return result; } From 914d4b8ca378957f522a9f0d551b22fd388ab333 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 12 Sep 2024 14:00:40 +0200 Subject: [PATCH 14/32] ranemed latency to metrics --- .../src/main/resources/thingsboard.yml | 4 +- ...AbstractMqttTimeseriesIntegrationTest.java | 47 +++++++++---------- .../data/device/profile/MqttTopics.java | 2 +- .../transport/mqtt/MqttTransportContext.java | 4 +- .../transport/mqtt/MqttTransportHandler.java | 2 +- ...ervice.java => GatewayMetricsService.java} | 36 +++++++------- .../GatewayMetricsData.java} | 4 +- .../GatewayMetricsState.java} | 38 +++++++-------- .../mqtt/session/GatewaySessionHandler.java | 14 +++--- .../src/main/resources/tb-mqtt-transport.yml | 2 + 10 files changed, 78 insertions(+), 75 deletions(-) rename common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/{GatewayLatencyService.java => GatewayMetricsService.java} (73%) rename common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/{latency/GatewayLatencyData.java => metrics/GatewayMetricsData.java} (83%) rename common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/{latency/GatewayLatencyState.java => metrics/GatewayMetricsState.java} (71%) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2fc35d915a..6b61838918 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1020,8 +1020,8 @@ transport: # MQTT disconnect timeout in milliseconds. The time to wait for the client to disconnect after the server sends a disconnect message. disconnect_timeout: "${MQTT_DISCONNECT_TIMEOUT:1000}" msg_queue_size_per_device_limit: "${MQTT_MSG_QUEUE_SIZE_PER_DEVICE_LIMIT:100}" # messages await in the queue before the device connected state. This limit works on the low level before TenantProfileLimits mechanism - # Interval of periodic report of the gateway latency - gateway_latency_report_interval_sec: "${MQTT_GATEWAY_LATENCY_REPORT_INTERVAL_SEC:3600}" + # Interval of periodic report of the gateway metrics + gateway_metrics_report_interval_sec: "${MQTT_GATEWAY_METRICS_REPORT_INTERVAL_SEC:3600}" netty: # Netty leak detector level leak_detector_level: "${NETTY_LEAK_DETECTOR_LVL:DISABLED}" diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index 886cf1e309..4f89a8018e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -25,13 +25,12 @@ import org.mockito.ArgumentCaptor; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest; import org.thingsboard.server.transport.mqtt.MqttTestConfigProperties; -import org.thingsboard.server.transport.mqtt.gateway.GatewayLatencyService; -import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyData; -import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyState; +import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService; +import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsData; +import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsState; import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestCallback; import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestClient; @@ -56,9 +55,9 @@ import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVIC import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_TELEMETRY_SHORT_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_TELEMETRY_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_CONNECT_TOPIC; -import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_LATENCY_TOPIC; +import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_METRICS_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_TELEMETRY_TOPIC; -import static org.thingsboard.server.transport.mqtt.mqttv3.credentials.BasicMqttCredentialsTest.CLIENT_ID; +import static org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService.METRICS_CHECK; @Slf4j public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqttIntegrationTest { @@ -70,7 +69,7 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt " \"key5\": {\"someNumber\": 42, \"someArray\": [1,2,3], \"someNestedObject\": {\"key\": \"value\"}}}"; @SpyBean - GatewayLatencyService gatewayLatencyService; + GatewayMetricsService gatewayMetricsService; @Before public void beforeTest() throws Exception { @@ -133,9 +132,9 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt } @Test - public void testPushLatencyGateway() throws Exception { + public void testPushMetricsGateway() throws Exception { MqttTestConfigProperties configProperties = MqttTestConfigProperties.builder() - .gatewayName("Test latency gateway") + .gatewayName("Test metrics gateway") .build(); processBeforeTest(configProperties); @@ -147,16 +146,16 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt publishLatency(client, gwLatencies, transportLatencies, 5); - gatewayLatencyService.reportLatency(); + gatewayMetricsService.reportMetrics(); - List actualKeys = getActualKeysList(savedGateway.getId(), List.of("latencyCheck")); - assertEquals("latencyCheck", actualKeys.get(0)); + List actualKeys = getActualKeysList(savedGateway.getId(), List.of(METRICS_CHECK)); + assertEquals(METRICS_CHECK, actualKeys.get(0)); - String telemetryUrl = String.format("/api/plugins/telemetry/DEVICE/%s/values/timeseries?startTs=%d&endTs=%d&keys=latencyCheck", savedGateway.getId(), 0, System.currentTimeMillis()); + String telemetryUrl = String.format("/api/plugins/telemetry/DEVICE/%s/values/timeseries?startTs=%d&endTs=%d&keys=%s", savedGateway.getId(), 0, System.currentTimeMillis(), METRICS_CHECK); Map>> gatewayTelemetry = doGetAsyncTyped(telemetryUrl, new TypeReference<>() {}); - Map latencyCheckTelemetry = gatewayTelemetry.get("latencyCheck").get(0); - Map latencyCheckValue = JacksonUtil.fromString((String) latencyCheckTelemetry.get("value"), new TypeReference<>() {}); + Map latencyCheckTelemetry = gatewayTelemetry.get(METRICS_CHECK).get(0); + Map latencyCheckValue = JacksonUtil.fromString((String) latencyCheckTelemetry.get("value"), new TypeReference<>() {}); assertNotNull(latencyCheckValue); gwLatencies.forEach((connectorName, gwLatencyList) -> { @@ -171,7 +170,7 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt long minTransportLatency = transportLatencyList.stream().mapToLong(Long::longValue).min().getAsLong(); long maxTransportLatency = transportLatencyList.stream().mapToLong(Long::longValue).max().getAsLong(); - GatewayLatencyState.ConnectorLatencyResult connectorLatencyResult = latencyCheckValue.get(connectorName); + GatewayMetricsState.ConnectorMetricsResult connectorLatencyResult = latencyCheckValue.get(connectorName); assertNotNull(connectorLatencyResult); checkConnectorLatencyResult(connectorLatencyResult, avgGwLatency, minGwLatency, maxGwLatency, avgTransportLatency, minTransportLatency, maxTransportLatency); }); @@ -180,27 +179,27 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt Awaitility.await() .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> verify(gatewayLatencyService).onDeviceDisconnect(savedGateway.getId())); + .untilAsserted(() -> verify(gatewayMetricsService).onDeviceDisconnect(savedGateway.getId())); } private void publishLatency(MqttTestClient client, Map> gwLatencies, Map> transportLatencies, int n) throws Exception { Random random = new Random(); for (int i = 0; i < n; i++) { - Map data = new HashMap<>(); + Map data = new HashMap<>(); long publishedTs = System.currentTimeMillis() - 10; long gatewayLatencyA = random.nextLong(100, 500); - data.put("connectorA", new GatewayLatencyData(publishedTs - gatewayLatencyA, publishedTs)); + data.put("connectorA", new GatewayMetricsData(publishedTs - gatewayLatencyA, publishedTs)); gwLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(gatewayLatencyA); boolean sendB = i % 2 == 0; if (sendB) { long gatewayLatencyB = random.nextLong(120, 450); - data.put("connectorB", new GatewayLatencyData(publishedTs - gatewayLatencyB, publishedTs)); + data.put("connectorB", new GatewayMetricsData(publishedTs - gatewayLatencyB, publishedTs)); gwLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(gatewayLatencyB); } - client.publishAndWait(GATEWAY_LATENCY_TOPIC, JacksonUtil.writeValueAsBytes(data)); + client.publishAndWait(GATEWAY_METRICS_TOPIC, JacksonUtil.writeValueAsBytes(data)); ArgumentCaptor transportReceiveTsCaptor = ArgumentCaptor.forClass(Long.class); - verify(gatewayLatencyService).process(any(), eq(savedGateway.getId()), eq(data), transportReceiveTsCaptor.capture()); + verify(gatewayMetricsService).process(any(), eq(savedGateway.getId()), eq(data), transportReceiveTsCaptor.capture()); Long transportReceiveTs = transportReceiveTsCaptor.getValue(); Long transportLatency = transportReceiveTs - publishedTs; transportLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(transportLatency); @@ -210,13 +209,13 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt } } - private void checkConnectorLatencyResult(GatewayLatencyState.ConnectorLatencyResult result, long avgGwLatency, long minGwLatency, long maxGwLatency, + private void checkConnectorLatencyResult(GatewayMetricsState.ConnectorMetricsResult result, long avgGwLatency, long minGwLatency, long maxGwLatency, long avgTransportLatency, long minTransportLatency, long maxTransportLatency) { assertNotNull(result); assertEquals(avgGwLatency, result.avgGwLatency()); assertEquals(minGwLatency, result.minGwLatency()); assertEquals(maxGwLatency, result.maxGwLatency()); - assertEquals(avgTransportLatency, result.transportLatencyAvg()); + assertEquals(avgTransportLatency, result.avgTransportLatency()); assertEquals(minTransportLatency, result.minTransportLatency()); assertEquals(maxTransportLatency, result.maxTransportLatency()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java index 8945f2d084..0590d57034 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java @@ -75,7 +75,7 @@ public class MqttTopics { public static final String GATEWAY_RPC_TOPIC = BASE_GATEWAY_API_TOPIC + RPC; public static final String GATEWAY_ATTRIBUTES_REQUEST_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_REQUEST; public static final String GATEWAY_ATTRIBUTES_RESPONSE_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_RESPONSE; - public static final String GATEWAY_LATENCY_TOPIC = BASE_GATEWAY_API_TOPIC + "/latency"; + public static final String GATEWAY_METRICS_TOPIC = BASE_GATEWAY_API_TOPIC + "/metrics"; // v2 topics public static final String BASE_DEVICE_API_TOPIC_V2 = "v2"; public static final String REQUEST_ID_PATTERN = "(?\\d+)"; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index dc61a47d78..d78a4afbc2 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.transport.mqtt.adaptors.JsonMqttAdaptor; import org.thingsboard.server.transport.mqtt.adaptors.ProtoMqttAdaptor; -import org.thingsboard.server.transport.mqtt.gateway.GatewayLatencyService; +import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService; import java.net.InetSocketAddress; import java.util.concurrent.atomic.AtomicInteger; @@ -58,7 +58,7 @@ public class MqttTransportContext extends TransportContext { @Getter @Autowired - private GatewayLatencyService gatewayLatencyService; + private GatewayMetricsService gatewayMetricsService; @Getter @Value("${transport.mqtt.netty.max_payload_size}") diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 4ae0d54a6b..1eee03f722 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -423,7 +423,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_DISCONNECT_TOPIC: gatewaySessionHandler.onDeviceDisconnect(mqttMsg); break; - case MqttTopics.GATEWAY_LATENCY_TOPIC: + case MqttTopics.GATEWAY_METRICS_TOPIC: gatewaySessionHandler.onGatewayLatency(mqttMsg); break; default: diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java similarity index 73% rename from common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java rename to common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java index acf10a03b8..263888f4c2 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayLatencyService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java @@ -27,8 +27,8 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.transport.mqtt.TbMqttTransportComponent; -import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyData; -import org.thingsboard.server.transport.mqtt.gateway.latency.GatewayLatencyState; +import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsData; +import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsState; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -37,10 +37,12 @@ import java.util.concurrent.TimeUnit; @Slf4j @Service @TbMqttTransportComponent -public class GatewayLatencyService { +public class GatewayMetricsService { - @Value("${transport.mqtt.gateway_latency_report_interval_sec:3600}") - private int latencyReportIntervalSec; + public static final String METRICS_CHECK = "metricsCheck"; + + @Value("${transport.mqtt.gateway_metrics_report_interval_sec:3600}") + private int metricsReportIntervalSec; @Autowired private SchedulerComponent scheduler; @@ -48,15 +50,15 @@ public class GatewayLatencyService { @Autowired private TransportService transportService; - private Map states = new ConcurrentHashMap<>(); + private Map states = new ConcurrentHashMap<>(); @PostConstruct private void init() { - scheduler.scheduleAtFixedRate(this::reportLatency, latencyReportIntervalSec, latencyReportIntervalSec, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(this::reportMetrics, metricsReportIntervalSec, metricsReportIntervalSec, TimeUnit.SECONDS); } - public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, Map data, long ts) { - states.computeIfAbsent(gatewayId, k -> new GatewayLatencyState(sessionInfo)).update(ts, data); + public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, Map data, long ts) { + states.computeIfAbsent(gatewayId, k -> new GatewayMetricsState(sessionInfo)).update(ts, data); } public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId) { @@ -71,34 +73,34 @@ public class GatewayLatencyService { } public void onDeviceDisconnect(DeviceId deviceId) { - GatewayLatencyState state = states.remove(deviceId); + GatewayMetricsState state = states.remove(deviceId); if (state != null) { - reportLatency(state, System.currentTimeMillis()); + reportMetrics(state, System.currentTimeMillis()); } } - public void reportLatency() { + public void reportMetrics() { if (states.isEmpty()) { return; } - Map oldStates = states; + Map oldStates = states; states = new ConcurrentHashMap<>(); long ts = System.currentTimeMillis(); oldStates.forEach((gatewayId, state) -> { - reportLatency(state, ts); + reportMetrics(state, ts); }); oldStates.clear(); } - private void reportLatency(GatewayLatencyState state, long ts) { + private void reportMetrics(GatewayMetricsState state, long ts) { if (state.isEmpty()) { return; } - var result = state.getLatencyStateResult(); + var result = state.getStateResult(); var kvProto = TransportProtos.KeyValueProto.newBuilder() - .setKey("latencyCheck") + .setKey(METRICS_CHECK) .setType(TransportProtos.KeyValueType.JSON_V) .setJsonV(JacksonUtil.toString(result)) .build(); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java similarity index 83% rename from common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java rename to common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java index 2cae428a02..23cc730b86 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyData.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.mqtt.gateway.latency; +package org.thingsboard.server.transport.mqtt.gateway.metrics; -public record GatewayLatencyData(long receivedTs, long publishedTs) { +public record GatewayMetricsData(long receivedTs, long publishedTs) { } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java similarity index 71% rename from common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java rename to common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java index 3ad75fee75..45f945151b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/latency/GatewayLatencyState.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.mqtt.gateway.latency; +package org.thingsboard.server.transport.mqtt.gateway.metrics; import lombok.Getter; import org.thingsboard.server.gen.transport.TransportProtos; @@ -25,15 +25,15 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -public class GatewayLatencyState { +public class GatewayMetricsState { - private final Map connectors; + private final Map connectors; private final Lock updateLock; @Getter private volatile TransportProtos.SessionInfoProto sessionInfo; - public GatewayLatencyState(TransportProtos.SessionInfoProto sessionInfo) { + public GatewayMetricsState(TransportProtos.SessionInfoProto sessionInfo) { this.connectors = new HashMap<>(); this.updateLock = new ReentrantLock(); this.sessionInfo = sessionInfo; @@ -43,19 +43,19 @@ public class GatewayLatencyState { this.sessionInfo = sessionInfo; } - public void update(long ts, Map latencyData) { + public void update(long ts, Map metricsData) { updateLock.lock(); try { - latencyData.forEach((connectorName, data) -> { - connectors.computeIfAbsent(connectorName, k -> new ConnectorLatencyState()).update(ts, data); + metricsData.forEach((connectorName, data) -> { + connectors.computeIfAbsent(connectorName, k -> new ConnectorMetricsState()).update(ts, data); }); } finally { updateLock.unlock(); } } - public Map getLatencyStateResult() { - Map result = new HashMap<>(); + public Map getStateResult() { + Map result = new HashMap<>(); updateLock.lock(); try { connectors.forEach((name, state) -> result.put(name, state.getResult())); @@ -71,7 +71,7 @@ public class GatewayLatencyState { return connectors.isEmpty(); } - private static class ConnectorLatencyState { + private static class ConnectorMetricsState { private final AtomicInteger count; private final AtomicLong gwLatencySum; private final AtomicLong transportLatencySum; @@ -80,15 +80,15 @@ public class GatewayLatencyState { private volatile long minTransportLatency; private volatile long maxTransportLatency; - private ConnectorLatencyState() { + private ConnectorMetricsState() { this.count = new AtomicInteger(0); this.gwLatencySum = new AtomicLong(0); this.transportLatencySum = new AtomicLong(0); } - private void update(long serverReceiveTs, GatewayLatencyData latencyData) { - long gwLatency = latencyData.publishedTs() - latencyData.receivedTs(); - long transportLatency = serverReceiveTs - latencyData.publishedTs(); + private void update(long serverReceiveTs, GatewayMetricsData metricsData) { + long gwLatency = metricsData.publishedTs() - metricsData.receivedTs(); + long transportLatency = serverReceiveTs - metricsData.publishedTs(); count.incrementAndGet(); gwLatencySum.addAndGet(gwLatency); transportLatencySum.addAndGet(transportLatency); @@ -106,16 +106,16 @@ public class GatewayLatencyState { } } - private ConnectorLatencyResult getResult() { + private ConnectorMetricsResult getResult() { long count = this.count.get(); long avgGwLatency = gwLatencySum.get() / count; - long transportLatencyAvg = transportLatencySum.get() / count; - return new ConnectorLatencyResult(avgGwLatency, minGwLatency, maxGwLatency, transportLatencyAvg, minTransportLatency, maxTransportLatency); + long avgTransportLatency = transportLatencySum.get() / count; + return new ConnectorMetricsResult(avgGwLatency, minGwLatency, maxGwLatency, avgTransportLatency, minTransportLatency, maxTransportLatency); } } - public record ConnectorLatencyResult(long avgGwLatency, long minGwLatency, long maxGwLatency, - long transportLatencyAvg, long minTransportLatency, long maxTransportLatency) { + public record ConnectorMetricsResult(long avgGwLatency, long minGwLatency, long maxGwLatency, + long avgTransportLatency, long minTransportLatency, long maxTransportLatency) { } } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index a7634593ce..32eaf6e0d7 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.transport.mqtt.gateway.GatewayLatencyService; +import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService; import java.util.Optional; import java.util.UUID; @@ -40,11 +40,11 @@ import static com.amazonaws.util.StringUtils.UTF8; @Slf4j public class GatewaySessionHandler extends AbstractGatewaySessionHandler { - private final GatewayLatencyService latencyService; + private final GatewayMetricsService gatewayMetricsService; public GatewaySessionHandler(DeviceSessionCtx deviceSessionCtx, UUID sessionId, boolean overwriteDevicesActivity) { super(deviceSessionCtx, sessionId, overwriteDevicesActivity); - this.latencyService = deviceSessionCtx.getContext().getGatewayLatencyService(); + this.gatewayMetricsService = deviceSessionCtx.getContext().getGatewayMetricsService(); } public void onDeviceConnect(MqttPublishMessage mqttMsg) throws AdaptorException { @@ -72,16 +72,16 @@ public class GatewaySessionHandler extends AbstractGatewaySessionHandler deviceProfileOpt) { this.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); - latencyService.onDeviceUpdate(sessionInfo, gateway.getDeviceId()); + gatewayMetricsService.onDeviceUpdate(sessionInfo, gateway.getDeviceId()); } public void onGatewayDelete(DeviceId deviceId) { - latencyService.onDeviceDelete(deviceId); + gatewayMetricsService.onDeviceDelete(deviceId); } public void onGatewayLatency(MqttPublishMessage mqttMsg) throws AdaptorException { @@ -94,7 +94,7 @@ public class GatewaySessionHandler extends AbstractGatewaySessionHandler() {}), ts); + gatewayMetricsService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), JacksonUtil.fromString(payload, new TypeReference<>() {}), ts); ack(msgId, MqttReasonCodes.PubAck.SUCCESS); } catch (Throwable t) { ackOrClose(msgId); diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 140839ccc3..d2ac105623 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -146,6 +146,8 @@ transport: # MQTT disconnect timeout in milliseconds. The time to wait for the client to disconnect after the server sends a disconnect message. disconnect_timeout: "${MQTT_DISCONNECT_TIMEOUT:1000}" msg_queue_size_per_device_limit: "${MQTT_MSG_QUEUE_SIZE_PER_DEVICE_LIMIT:100}" # messages await in the queue before device connected state. This limit works on low level before TenantProfileLimits mechanism + # Interval of periodic report of the gateway metrics + gateway_metrics_report_interval_sec: "${MQTT_GATEWAY_METRICS_REPORT_INTERVAL_SEC:3600}" netty: # Netty leak detector level leak_detector_level: "${NETTY_LEAK_DETECTOR_LVL:DISABLED}" From e65fc9d55e881f5852fff1e6032d72926584a97b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 12 Sep 2024 15:16:32 +0200 Subject: [PATCH 15/32] minor refactoring due to comments --- .../AbstractMqttTimeseriesIntegrationTest.java | 10 +++++----- .../server/transport/mqtt/MqttTransportHandler.java | 2 +- .../transport/mqtt/gateway/GatewayMetricsService.java | 4 ++-- .../transport/mqtt/session/GatewaySessionHandler.java | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index 4f89a8018e..32854a8735 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -57,7 +57,7 @@ import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVIC import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_CONNECT_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_METRICS_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_TELEMETRY_TOPIC; -import static org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService.METRICS_CHECK; +import static org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService.GATEWAY_METRICS; @Slf4j public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqttIntegrationTest { @@ -148,13 +148,13 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt gatewayMetricsService.reportMetrics(); - List actualKeys = getActualKeysList(savedGateway.getId(), List.of(METRICS_CHECK)); - assertEquals(METRICS_CHECK, actualKeys.get(0)); + List actualKeys = getActualKeysList(savedGateway.getId(), List.of(GATEWAY_METRICS)); + assertEquals(GATEWAY_METRICS, actualKeys.get(0)); - String telemetryUrl = String.format("/api/plugins/telemetry/DEVICE/%s/values/timeseries?startTs=%d&endTs=%d&keys=%s", savedGateway.getId(), 0, System.currentTimeMillis(), METRICS_CHECK); + String telemetryUrl = String.format("/api/plugins/telemetry/DEVICE/%s/values/timeseries?startTs=%d&endTs=%d&keys=%s", savedGateway.getId(), 0, System.currentTimeMillis(), GATEWAY_METRICS); Map>> gatewayTelemetry = doGetAsyncTyped(telemetryUrl, new TypeReference<>() {}); - Map latencyCheckTelemetry = gatewayTelemetry.get(METRICS_CHECK).get(0); + Map latencyCheckTelemetry = gatewayTelemetry.get(GATEWAY_METRICS).get(0); Map latencyCheckValue = JacksonUtil.fromString((String) latencyCheckTelemetry.get("value"), new TypeReference<>() {}); assertNotNull(latencyCheckValue); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 1eee03f722..7fad363c6b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -424,7 +424,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement gatewaySessionHandler.onDeviceDisconnect(mqttMsg); break; case MqttTopics.GATEWAY_METRICS_TOPIC: - gatewaySessionHandler.onGatewayLatency(mqttMsg); + gatewaySessionHandler.onGatewayMetrics(mqttMsg); break; default: ack(ctx, msgId, MqttReasonCodes.PubAck.TOPIC_NAME_INVALID); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java index 263888f4c2..c044c89e15 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java @@ -39,7 +39,7 @@ import java.util.concurrent.TimeUnit; @TbMqttTransportComponent public class GatewayMetricsService { - public static final String METRICS_CHECK = "metricsCheck"; + public static final String GATEWAY_METRICS = "gatewayMetrics"; @Value("${transport.mqtt.gateway_metrics_report_interval_sec:3600}") private int metricsReportIntervalSec; @@ -100,7 +100,7 @@ public class GatewayMetricsService { } var result = state.getStateResult(); var kvProto = TransportProtos.KeyValueProto.newBuilder() - .setKey(METRICS_CHECK) + .setKey(GATEWAY_METRICS) .setType(TransportProtos.KeyValueType.JSON_V) .setJsonV(JacksonUtil.toString(result)) .build(); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index 32eaf6e0d7..9444787193 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -84,7 +84,7 @@ public class GatewaySessionHandler extends AbstractGatewaySessionHandler Date: Thu, 12 Sep 2024 22:28:10 +0200 Subject: [PATCH 16/32] implemented telemetry metadata instead of separate topic for metrics --- .../src/main/resources/thingsboard.yml | 2 +- ...AbstractMqttTimeseriesIntegrationTest.java | 75 ++++++++++--------- .../data/device/profile/MqttTopics.java | 1 - .../server/common/adaptor/JsonConverter.java | 20 ++++- .../transport/mqtt/MqttTransportHandler.java | 5 +- .../mqtt/gateway/GatewayMetricsService.java | 19 ++--- .../gateway/metrics/GatewayMetricsData.java | 2 +- .../gateway/metrics/GatewayMetricsState.java | 9 ++- .../AbstractGatewaySessionHandler.java | 22 +++++- .../mqtt/session/GatewaySessionHandler.java | 31 -------- .../src/main/resources/tb-mqtt-transport.yml | 2 +- 11 files changed, 92 insertions(+), 96 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6b61838918..718c4823c4 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1021,7 +1021,7 @@ transport: disconnect_timeout: "${MQTT_DISCONNECT_TIMEOUT:1000}" msg_queue_size_per_device_limit: "${MQTT_MSG_QUEUE_SIZE_PER_DEVICE_LIMIT:100}" # messages await in the queue before the device connected state. This limit works on the low level before TenantProfileLimits mechanism # Interval of periodic report of the gateway metrics - gateway_metrics_report_interval_sec: "${MQTT_GATEWAY_METRICS_REPORT_INTERVAL_SEC:3600}" + gateway_metrics_report_interval_sec: "${MQTT_GATEWAY_METRICS_REPORT_INTERVAL_SEC:60}" netty: # Netty leak detector level leak_detector_level: "${NETTY_LEAK_DETECTOR_LVL:DISABLED}" diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index 32854a8735..f5e21e9e18 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.transport.mqtt.mqttv3.telemetry.timeseries; import com.fasterxml.jackson.core.type.TypeReference; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; -import org.awaitility.Awaitility; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -55,7 +54,6 @@ import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVIC import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_TELEMETRY_SHORT_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_TELEMETRY_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_CONNECT_TOPIC; -import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_METRICS_TOPIC; import static org.thingsboard.server.common.data.device.profile.MqttTopics.GATEWAY_TELEMETRY_TOPIC; import static org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService.GATEWAY_METRICS; @@ -124,8 +122,8 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt String deviceName = "Device A"; Device device = doExecuteWithRetriesAndInterval(() -> doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class), - 20, - 100); + 20, + 100); assertNotNull(device); client.disconnect(); @@ -138,13 +136,10 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt .build(); processBeforeTest(configProperties); - MqttTestClient client = new MqttTestClient(); - client.connectAndWait(gatewayAccessToken); - Map> gwLatencies = new HashMap<>(); Map> transportLatencies = new HashMap<>(); - publishLatency(client, gwLatencies, transportLatencies, 5); + publishLatency(gwLatencies, transportLatencies, 5); gatewayMetricsService.reportMetrics(); @@ -174,38 +169,37 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt assertNotNull(connectorLatencyResult); checkConnectorLatencyResult(connectorLatencyResult, avgGwLatency, minGwLatency, maxGwLatency, avgTransportLatency, minTransportLatency, maxTransportLatency); }); - - client.disconnect(); - - Awaitility.await() - .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> verify(gatewayMetricsService).onDeviceDisconnect(savedGateway.getId())); } - private void publishLatency(MqttTestClient client, Map> gwLatencies, Map> transportLatencies, int n) throws Exception { + private void publishLatency(Map> gwLatencies, Map> transportLatencies, int n) throws Exception { Random random = new Random(); for (int i = 0; i < n; i++) { - Map data = new HashMap<>(); long publishedTs = System.currentTimeMillis() - 10; long gatewayLatencyA = random.nextLong(100, 500); - data.put("connectorA", new GatewayMetricsData(publishedTs - gatewayLatencyA, publishedTs)); + var firstData = new GatewayMetricsData("connectorA", publishedTs - gatewayLatencyA, publishedTs); gwLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(gatewayLatencyA); - boolean sendB = i % 2 == 0; - if (sendB) { - long gatewayLatencyB = random.nextLong(120, 450); - data.put("connectorB", new GatewayMetricsData(publishedTs - gatewayLatencyB, publishedTs)); - gwLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(gatewayLatencyB); - } - - client.publishAndWait(GATEWAY_METRICS_TOPIC, JacksonUtil.writeValueAsBytes(data)); - ArgumentCaptor transportReceiveTsCaptor = ArgumentCaptor.forClass(Long.class); - verify(gatewayMetricsService).process(any(), eq(savedGateway.getId()), eq(data), transportReceiveTsCaptor.capture()); - Long transportReceiveTs = transportReceiveTsCaptor.getValue(); - Long transportLatency = transportReceiveTs - publishedTs; - transportLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(transportLatency); - if (sendB) { - transportLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(transportLatency); - } + long gatewayLatencyB = random.nextLong(120, 450); + var secondData = new GatewayMetricsData("connectorB", publishedTs - gatewayLatencyB, publishedTs); + gwLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(gatewayLatencyB); + + List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5"); + String deviceName1 = "Device A"; + String deviceName2 = "Device B"; + String firstMetadata = JacksonUtil.writeValueAsString(firstData); + String secondMetadata = JacksonUtil.writeValueAsString(secondData); + String payload = getGatewayTelemetryJsonPayloadWithMetadata(deviceName1, deviceName2, "10000", "20000", firstMetadata, secondMetadata); + processGatewayTelemetryTest(GATEWAY_TELEMETRY_TOPIC, expectedKeys, payload.getBytes(), deviceName1, deviceName2); + + ArgumentCaptor transportReceiveTsCaptorA = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor transportReceiveTsCaptorB = ArgumentCaptor.forClass(Long.class); + verify(gatewayMetricsService).process(any(), eq(savedGateway.getId()), eq(List.of(firstData)), transportReceiveTsCaptorA.capture()); + verify(gatewayMetricsService).process(any(), eq(savedGateway.getId()), eq(List.of(secondData)), transportReceiveTsCaptorB.capture()); + Long transportReceiveTsA = transportReceiveTsCaptorA.getValue(); + Long transportReceiveTsB = transportReceiveTsCaptorB.getValue(); + Long transportLatencyA = transportReceiveTsA - publishedTs; + Long transportLatencyB = transportReceiveTsB - publishedTs; + transportLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(transportLatencyA); + transportLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(transportLatencyB); } } @@ -250,7 +244,8 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt long end = System.currentTimeMillis() + 5000; Map>> values = null; while (start <= end) { - values = doGetAsyncTyped(getTelemetryValuesUrl, new TypeReference<>() {}); + values = doGetAsyncTyped(getTelemetryValuesUrl, new TypeReference<>() { + }); boolean valid = values.size() == expectedKeys.size(); if (valid) { for (String key : expectedKeys) { @@ -289,7 +284,7 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt MqttTestClient client = new MqttTestClient(); client.connectAndWait(gatewayAccessToken); client.publishAndWait(topic, payload); - client.disconnect(); + client.disconnectAndWait(); Device firstDevice = doExecuteWithRetriesAndInterval(() -> doGet("/api/tenant/devices?deviceName=" + firstDeviceName, Device.class), 20, @@ -346,6 +341,16 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt return "{\"" + deviceA + "\": " + payload + ", \"" + deviceB + "\": " + payload + "}"; } + protected String getGatewayTelemetryJsonPayloadWithMetadata(String deviceA, String deviceB, String firstTsValue, String secondTsValue, String firstMetadata, String secondMetadata) { + String payloadA = "[{\"ts\": " + firstTsValue + ", \"values\": " + PAYLOAD_VALUES_STR + ", \"metadata\":" + firstMetadata + "}, " + + "{\"ts\": " + secondTsValue + ", \"values\": " + PAYLOAD_VALUES_STR + "}]"; + + String payloadB = "[{\"ts\": " + firstTsValue + ", \"values\": " + PAYLOAD_VALUES_STR + ", \"metadata\":" + secondMetadata + "}, " + + "{\"ts\": " + secondTsValue + ", \"values\": " + PAYLOAD_VALUES_STR + "}]"; + + return "{\"" + deviceA + "\": " + payloadA + ", \"" + deviceB + "\": " + payloadB + "}"; + } + private String getTelemetryValuesUrl(DeviceId deviceId, Set actualKeySet) { return "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?startTs=0&endTs=25000&keys=" + String.join(",", actualKeySet); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java index 0590d57034..cabdea145c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java @@ -75,7 +75,6 @@ public class MqttTopics { public static final String GATEWAY_RPC_TOPIC = BASE_GATEWAY_API_TOPIC + RPC; public static final String GATEWAY_ATTRIBUTES_REQUEST_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_REQUEST; public static final String GATEWAY_ATTRIBUTES_RESPONSE_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_RESPONSE; - public static final String GATEWAY_METRICS_TOPIC = BASE_GATEWAY_API_TOPIC + "/metrics"; // v2 topics public static final String BASE_DEVICE_API_TOPIC_V2 = "v2"; public static final String REQUEST_ID_PATTERN = "(?\\d+)"; diff --git a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java index f5c7bbb901..6a61531fec 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java @@ -74,22 +74,34 @@ public class JsonConverter { private static int maxStringValueLength = 0; public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement, long ts) throws JsonSyntaxException { + return convertToTelemetryProto(jsonElement, ts, null); + } + + public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement, long ts, List metadataResult) throws JsonSyntaxException { PostTelemetryMsg.Builder builder = PostTelemetryMsg.newBuilder(); - convertToTelemetry(jsonElement, ts, null, builder); + convertToTelemetry(jsonElement, ts, null, builder, metadataResult); return builder.build(); } + public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement, List metadataResult) throws JsonSyntaxException { + return convertToTelemetryProto(jsonElement, System.currentTimeMillis(), metadataResult); + } + public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement) throws JsonSyntaxException { return convertToTelemetryProto(jsonElement, System.currentTimeMillis()); } - private static void convertToTelemetry(JsonElement jsonElement, long systemTs, Map> result, PostTelemetryMsg.Builder builder) { + private static void convertToTelemetry(JsonElement jsonElement, long systemTs, Map> result, PostTelemetryMsg.Builder builder, List metadataResult) { if (jsonElement.isJsonObject()) { parseObject(systemTs, result, builder, jsonElement.getAsJsonObject()); } else if (jsonElement.isJsonArray()) { jsonElement.getAsJsonArray().forEach(je -> { if (je.isJsonObject()) { - parseObject(systemTs, result, builder, je.getAsJsonObject()); + JsonObject jo = je.getAsJsonObject(); + if (metadataResult != null && jo.has("metadata")) { + metadataResult.add(jo.get("metadata")); + } + parseObject(systemTs, result, builder, jo); } else { throw new JsonSyntaxException(CAN_T_PARSE_VALUE + je); } @@ -550,7 +562,7 @@ public class JsonConverter { public static Map> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted) throws JsonSyntaxException { Map> result = sorted ? new TreeMap<>() : new HashMap<>(); - convertToTelemetry(jsonElement, systemTs, result, null); + convertToTelemetry(jsonElement, systemTs, result, null, null); return result; } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 7fad363c6b..d7f64e58ee 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -423,9 +423,6 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_DISCONNECT_TOPIC: gatewaySessionHandler.onDeviceDisconnect(mqttMsg); break; - case MqttTopics.GATEWAY_METRICS_TOPIC: - gatewaySessionHandler.onGatewayMetrics(mqttMsg); - break; default: ack(ctx, msgId, MqttReasonCodes.PubAck.TOPIC_NAME_INVALID); } @@ -1199,7 +1196,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement transportService.process(deviceSessionCtx.getSessionInfo(), SESSION_EVENT_MSG_CLOSED, null); transportService.deregisterSession(deviceSessionCtx.getSessionInfo()); if (gatewaySessionHandler != null) { - gatewaySessionHandler.onGatewayDisconnect(); + gatewaySessionHandler.onDevicesDisconnect(); } if (sparkplugSessionHandler != null) { // add Msg Telemetry node: key STATE type: String value: OFFLINE ts: sparkplugBProto.getTimestamp() diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java index c044c89e15..18f0047288 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.transport.mqtt.TbMqttTransportComponent; import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsData; import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsState; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -41,7 +42,7 @@ public class GatewayMetricsService { public static final String GATEWAY_METRICS = "gatewayMetrics"; - @Value("${transport.mqtt.gateway_metrics_report_interval_sec:3600}") + @Value("${transport.mqtt.gateway_metrics_report_interval_sec:60}") private int metricsReportIntervalSec; @Autowired @@ -57,8 +58,8 @@ public class GatewayMetricsService { scheduler.scheduleAtFixedRate(this::reportMetrics, metricsReportIntervalSec, metricsReportIntervalSec, TimeUnit.SECONDS); } - public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, Map data, long ts) { - states.computeIfAbsent(gatewayId, k -> new GatewayMetricsState(sessionInfo)).update(ts, data); + public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, List data, long serverReceiveTs) { + states.computeIfAbsent(gatewayId, k -> new GatewayMetricsState(sessionInfo)).update(data, serverReceiveTs); } public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId) { @@ -72,26 +73,18 @@ public class GatewayMetricsService { states.remove(deviceId); } - public void onDeviceDisconnect(DeviceId deviceId) { - GatewayMetricsState state = states.remove(deviceId); - if (state != null) { - reportMetrics(state, System.currentTimeMillis()); - } - } - public void reportMetrics() { if (states.isEmpty()) { return; } - Map oldStates = states; + Map statesToReport = states; states = new ConcurrentHashMap<>(); long ts = System.currentTimeMillis(); - oldStates.forEach((gatewayId, state) -> { + statesToReport.forEach((gatewayId, state) -> { reportMetrics(state, ts); }); - oldStates.clear(); } private void reportMetrics(GatewayMetricsState state, long ts) { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java index 23cc730b86..9829a1d373 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java @@ -15,5 +15,5 @@ */ package org.thingsboard.server.transport.mqtt.gateway.metrics; -public record GatewayMetricsData(long receivedTs, long publishedTs) { +public record GatewayMetricsData(String connector, long receivedTs, long publishedTs) { } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java index 45f945151b..7bec5da6a7 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java @@ -19,6 +19,7 @@ import lombok.Getter; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -43,11 +44,11 @@ public class GatewayMetricsState { this.sessionInfo = sessionInfo; } - public void update(long ts, Map metricsData) { + public void update(List metricsData, long serverReceiveTs) { updateLock.lock(); try { - metricsData.forEach((connectorName, data) -> { - connectors.computeIfAbsent(connectorName, k -> new ConnectorMetricsState()).update(ts, data); + metricsData.forEach(data -> { + connectors.computeIfAbsent(data.connector(), k -> new ConnectorMetricsState()).update(data, serverReceiveTs); }); } finally { updateLock.unlock(); @@ -86,7 +87,7 @@ public class GatewayMetricsState { this.transportLatencySum = new AtomicLong(0); } - private void update(long serverReceiveTs, GatewayMetricsData metricsData) { + private void update(GatewayMetricsData metricsData, long serverReceiveTs) { long gwLatency = metricsData.publishedTs() - metricsData.receivedTs(); long transportLatency = serverReceiveTs - metricsData.publishedTs(); count.incrementAndGet(); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index 3b3aaef983..f1728a2eac 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -62,6 +62,8 @@ import org.thingsboard.server.transport.mqtt.MqttTransportHandler; import org.thingsboard.server.transport.mqtt.adaptors.JsonMqttAdaptor; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; import org.thingsboard.server.transport.mqtt.adaptors.ProtoMqttAdaptor; +import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService; +import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsData; import org.thingsboard.server.transport.mqtt.util.sparkplug.SparkplugConnectionState; import java.util.ArrayList; @@ -114,6 +116,7 @@ public abstract class AbstractGatewaySessionHandler mqttQoSMap; protected final ChannelHandlerContext channel; protected final DeviceSessionCtx deviceSessionCtx; + protected final GatewayMetricsService gatewayMetricsService; @Getter @Setter @@ -131,6 +134,7 @@ public abstract class AbstractGatewaySessionHandler createWeakMap() { @@ -380,7 +384,9 @@ public abstract class AbstractGatewaySessionHandler metadata = new ArrayList<>(); + TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(msg.getAsJsonArray(), metadata); + processTelemetryMetadataMsg(metadata); transportService.process(deviceCtx.getSessionInfo(), postTelemetryMsg, getPubAckCallback(channel, deviceName, msgId, postTelemetryMsg)); } catch (Throwable e) { log.warn("[{}][{}][{}] Failed to convert telemetry: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, msg, e); @@ -388,6 +394,20 @@ public abstract class AbstractGatewaySessionHandler metadata) { + var serverReceiveTs = System.currentTimeMillis(); + var metricsData = metadata.stream() + .filter(JsonElement::isJsonObject) + .map(je -> { + var jo = je.getAsJsonObject(); + var connector = jo.get("connector").getAsString(); + var receivedTs = jo.get("receivedTs").getAsLong(); + var publishedTs = jo.get("publishedTs").getAsLong(); + return new GatewayMetricsData(connector, receivedTs, publishedTs); + }).toList(); + gatewayMetricsService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), metricsData, serverReceiveTs); + } + protected void onDeviceTelemetryProto(int msgId, ByteBuf payload) throws AdaptorException { try { TransportApiProtos.GatewayTelemetryMsg telemetryMsgProto = TransportApiProtos.GatewayTelemetryMsg.parseFrom(getBytes(payload)); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index 9444787193..5dce8e8893 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -15,36 +15,27 @@ */ package org.thingsboard.server.transport.mqtt.session; -import com.fasterxml.jackson.core.type.TypeReference; import io.netty.buffer.ByteBuf; import io.netty.handler.codec.mqtt.MqttPublishMessage; -import io.netty.handler.codec.mqtt.MqttReasonCodes; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService; import java.util.Optional; import java.util.UUID; -import static com.amazonaws.util.StringUtils.UTF8; - /** * Created by nickAS21 on 26.12.22 */ @Slf4j public class GatewaySessionHandler extends AbstractGatewaySessionHandler { - private final GatewayMetricsService gatewayMetricsService; - public GatewaySessionHandler(DeviceSessionCtx deviceSessionCtx, UUID sessionId, boolean overwriteDevicesActivity) { super(deviceSessionCtx, sessionId, overwriteDevicesActivity); - this.gatewayMetricsService = deviceSessionCtx.getContext().getGatewayMetricsService(); } public void onDeviceConnect(MqttPublishMessage mqttMsg) throws AdaptorException { @@ -70,11 +61,6 @@ public class GatewaySessionHandler extends AbstractGatewaySessionHandler deviceProfileOpt) { this.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); gatewayMetricsService.onDeviceUpdate(sessionInfo, gateway.getDeviceId()); @@ -84,21 +70,4 @@ public class GatewaySessionHandler extends AbstractGatewaySessionHandler() {}), ts); - ack(msgId, MqttReasonCodes.PubAck.SUCCESS); - } catch (Throwable t) { - ackOrClose(msgId); - } - } - } diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index d2ac105623..c9fd10a99d 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -147,7 +147,7 @@ transport: disconnect_timeout: "${MQTT_DISCONNECT_TIMEOUT:1000}" msg_queue_size_per_device_limit: "${MQTT_MSG_QUEUE_SIZE_PER_DEVICE_LIMIT:100}" # messages await in the queue before device connected state. This limit works on low level before TenantProfileLimits mechanism # Interval of periodic report of the gateway metrics - gateway_metrics_report_interval_sec: "${MQTT_GATEWAY_METRICS_REPORT_INTERVAL_SEC:3600}" + gateway_metrics_report_interval_sec: "${MQTT_GATEWAY_METRICS_REPORT_INTERVAL_SEC:60}" netty: # Netty leak detector level leak_detector_level: "${NETTY_LEAK_DETECTOR_LVL:DISABLED}" From 55b6b8c5645e3fd9748ecae83f9b4800dc59baf3 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 13 Sep 2024 10:02:06 +0200 Subject: [PATCH 17/32] added one more filtering for metrics data --- .../mqtt/session/AbstractGatewaySessionHandler.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index f1728a2eac..f963a78135 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -398,14 +398,19 @@ public abstract class AbstractGatewaySessionHandler { - var jo = je.getAsJsonObject(); + .map(JsonElement::getAsJsonObject) + .filter(jo -> jo.has("connector") + && jo.has("receivedTs") + && jo.has("publishedTs")) + .map(jo -> { var connector = jo.get("connector").getAsString(); var receivedTs = jo.get("receivedTs").getAsLong(); var publishedTs = jo.get("publishedTs").getAsLong(); return new GatewayMetricsData(connector, receivedTs, publishedTs); }).toList(); - gatewayMetricsService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), metricsData, serverReceiveTs); + if (!metadata.isEmpty()) { + gatewayMetricsService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), metricsData, serverReceiveTs); + } } protected void onDeviceTelemetryProto(int msgId, ByteBuf payload) throws AdaptorException { From 4440e559feff9b2ea440eb13bc3da9b8e64e3120 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 13 Sep 2024 10:50:21 +0200 Subject: [PATCH 18/32] minor fix --- .../transport/mqtt/session/AbstractGatewaySessionHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index f963a78135..483522f85c 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -408,7 +408,7 @@ public abstract class AbstractGatewaySessionHandler Date: Fri, 13 Sep 2024 15:30:37 +0200 Subject: [PATCH 19/32] created separate converter methow for gateway telemetry --- ...AbstractMqttTimeseriesIntegrationTest.java | 6 +- .../msg/gateway/metrics/GatewayMetadata.java} | 4 +- .../server/common/adaptor/JsonConverter.java | 62 ++++++++++++++----- .../mqtt/gateway/GatewayMetricsService.java | 4 +- .../gateway/metrics/GatewayMetricsState.java | 5 +- .../AbstractGatewaySessionHandler.java | 32 +++------- 6 files changed, 67 insertions(+), 46 deletions(-) rename common/{transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java => message/src/main/java/org/thingsboard/server/common/msg/gateway/metrics/GatewayMetadata.java} (80%) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java index f5e21e9e18..5fd68417dd 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/telemetry/timeseries/AbstractMqttTimeseriesIntegrationTest.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest; import org.thingsboard.server.transport.mqtt.MqttTestConfigProperties; import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService; -import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsData; +import org.thingsboard.server.common.msg.gateway.metrics.GatewayMetadata; import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsState; import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestCallback; import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestClient; @@ -176,10 +176,10 @@ public abstract class AbstractMqttTimeseriesIntegrationTest extends AbstractMqtt for (int i = 0; i < n; i++) { long publishedTs = System.currentTimeMillis() - 10; long gatewayLatencyA = random.nextLong(100, 500); - var firstData = new GatewayMetricsData("connectorA", publishedTs - gatewayLatencyA, publishedTs); + var firstData = new GatewayMetadata("connectorA", publishedTs - gatewayLatencyA, publishedTs); gwLatencies.computeIfAbsent("connectorA", key -> new ArrayList<>()).add(gatewayLatencyA); long gatewayLatencyB = random.nextLong(120, 450); - var secondData = new GatewayMetricsData("connectorB", publishedTs - gatewayLatencyB, publishedTs); + var secondData = new GatewayMetadata("connectorB", publishedTs - gatewayLatencyB, publishedTs); gwLatencies.computeIfAbsent("connectorB", key -> new ArrayList<>()).add(gatewayLatencyB); List expectedKeys = Arrays.asList("key1", "key2", "key3", "key4", "key5"); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java b/common/message/src/main/java/org/thingsboard/server/common/msg/gateway/metrics/GatewayMetadata.java similarity index 80% rename from common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/gateway/metrics/GatewayMetadata.java index 9829a1d373..cc905f9383 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsData.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/gateway/metrics/GatewayMetadata.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.mqtt.gateway.metrics; +package org.thingsboard.server.common.msg.gateway.metrics; -public record GatewayMetricsData(String connector, long receivedTs, long publishedTs) { +public record GatewayMetadata(String connector, long receivedTs, long publishedTs) { } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java index 6a61531fec..bffe59fa83 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.gateway.metrics.GatewayMetadata; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; @@ -74,33 +76,65 @@ public class JsonConverter { private static int maxStringValueLength = 0; public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement, long ts) throws JsonSyntaxException { - return convertToTelemetryProto(jsonElement, ts, null); - } - - public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement, long ts, List metadataResult) throws JsonSyntaxException { PostTelemetryMsg.Builder builder = PostTelemetryMsg.newBuilder(); - convertToTelemetry(jsonElement, ts, null, builder, metadataResult); + convertToTelemetry(jsonElement, ts, null, builder); return builder.build(); } - public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement, List metadataResult) throws JsonSyntaxException { - return convertToTelemetryProto(jsonElement, System.currentTimeMillis(), metadataResult); - } - public static PostTelemetryMsg convertToTelemetryProto(JsonElement jsonElement) throws JsonSyntaxException { return convertToTelemetryProto(jsonElement, System.currentTimeMillis()); } - private static void convertToTelemetry(JsonElement jsonElement, long systemTs, Map> result, PostTelemetryMsg.Builder builder, List metadataResult) { + public static TbPair> convertToGatewayTelemetry(JsonElement jsonElement, long systemTs) { + List metadataResult = null; + PostTelemetryMsg.Builder builder = PostTelemetryMsg.newBuilder(); + if (jsonElement.isJsonArray()) { + var ja = jsonElement.getAsJsonArray(); + for (int i = 0; i < ja.size(); i++) { + var je = ja.get(i); + if (je.isJsonObject()) { + JsonObject jo = je.getAsJsonObject(); + JsonElement metadataElem = jo.remove("metadata"); + if (metadataElem != null) { + if (metadataResult == null) { + metadataResult = new ArrayList<>(); + } + if (metadataElem.isJsonObject()) { + JsonObject metadataObj = metadataElem.getAsJsonObject(); + var connector = getAndValidateMetadataElement(metadataObj, "connector").getAsString(); + var receivedTs = getAndValidateMetadataElement(metadataObj, "receivedTs").getAsLong(); + var publishedTs = getAndValidateMetadataElement(metadataObj, "publishedTs").getAsLong(); + metadataResult.add(new GatewayMetadata(connector, receivedTs, publishedTs)); + } else { + throw new JsonSyntaxException("Can't parse gateway metadata: " + metadataElem); + } + } + parseObject(systemTs, null, builder, jo); + } else { + throw new JsonSyntaxException(CAN_T_PARSE_VALUE + je); + } + } + } else { + throw new JsonSyntaxException(CAN_T_PARSE_VALUE + jsonElement); + } + return TbPair.of(builder.build(), metadataResult); + } + + private static JsonElement getAndValidateMetadataElement(JsonObject metadata, String elementName) { + var element = metadata.get(elementName); + if (element == null || element.isJsonNull()) { + throw new JsonSyntaxException(String.format("Can't parse gateway element in metadata: [%s][%s]", metadata, elementName)); + } + return element; + } + + private static void convertToTelemetry(JsonElement jsonElement, long systemTs, Map> result, PostTelemetryMsg.Builder builder) { if (jsonElement.isJsonObject()) { parseObject(systemTs, result, builder, jsonElement.getAsJsonObject()); } else if (jsonElement.isJsonArray()) { jsonElement.getAsJsonArray().forEach(je -> { if (je.isJsonObject()) { JsonObject jo = je.getAsJsonObject(); - if (metadataResult != null && jo.has("metadata")) { - metadataResult.add(jo.get("metadata")); - } parseObject(systemTs, result, builder, jo); } else { throw new JsonSyntaxException(CAN_T_PARSE_VALUE + je); @@ -562,7 +596,7 @@ public class JsonConverter { public static Map> convertToTelemetry(JsonElement jsonElement, long systemTs, boolean sorted) throws JsonSyntaxException { Map> result = sorted ? new TreeMap<>() : new HashMap<>(); - convertToTelemetry(jsonElement, systemTs, result, null, null); + convertToTelemetry(jsonElement, systemTs, result, null); return result; } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java index 18f0047288..4f67866d06 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/GatewayMetricsService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.transport.mqtt.TbMqttTransportComponent; -import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsData; +import org.thingsboard.server.common.msg.gateway.metrics.GatewayMetadata; import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsState; import java.util.List; @@ -58,7 +58,7 @@ public class GatewayMetricsService { scheduler.scheduleAtFixedRate(this::reportMetrics, metricsReportIntervalSec, metricsReportIntervalSec, TimeUnit.SECONDS); } - public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, List data, long serverReceiveTs) { + public void process(TransportProtos.SessionInfoProto sessionInfo, DeviceId gatewayId, List data, long serverReceiveTs) { states.computeIfAbsent(gatewayId, k -> new GatewayMetricsState(sessionInfo)).update(data, serverReceiveTs); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java index 7bec5da6a7..a800c78e28 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/gateway/metrics/GatewayMetricsState.java @@ -16,6 +16,7 @@ package org.thingsboard.server.transport.mqtt.gateway.metrics; import lombok.Getter; +import org.thingsboard.server.common.msg.gateway.metrics.GatewayMetadata; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.HashMap; @@ -44,7 +45,7 @@ public class GatewayMetricsState { this.sessionInfo = sessionInfo; } - public void update(List metricsData, long serverReceiveTs) { + public void update(List metricsData, long serverReceiveTs) { updateLock.lock(); try { metricsData.forEach(data -> { @@ -87,7 +88,7 @@ public class GatewayMetricsState { this.transportLatencySum = new AtomicLong(0); } - private void update(GatewayMetricsData metricsData, long serverReceiveTs) { + private void update(GatewayMetadata metricsData, long serverReceiveTs) { long gwLatency = metricsData.publishedTs() - metricsData.receivedTs(); long transportLatency = serverReceiveTs - metricsData.publishedTs(); count.incrementAndGet(); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index 483522f85c..2e205ff840 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -48,6 +48,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.gateway.metrics.GatewayMetadata; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; @@ -63,7 +65,6 @@ import org.thingsboard.server.transport.mqtt.adaptors.JsonMqttAdaptor; import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; import org.thingsboard.server.transport.mqtt.adaptors.ProtoMqttAdaptor; import org.thingsboard.server.transport.mqtt.gateway.GatewayMetricsService; -import org.thingsboard.server.transport.mqtt.gateway.metrics.GatewayMetricsData; import org.thingsboard.server.transport.mqtt.util.sparkplug.SparkplugConnectionState; import java.util.ArrayList; @@ -384,9 +385,13 @@ public abstract class AbstractGatewaySessionHandler metadata = new ArrayList<>(); - TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(msg.getAsJsonArray(), metadata); - processTelemetryMetadataMsg(metadata); + long systemTs = System.currentTimeMillis(); + TbPair> gatewayPayloadPair = JsonConverter.convertToGatewayTelemetry(msg.getAsJsonArray(), systemTs); + TransportProtos.PostTelemetryMsg postTelemetryMsg = gatewayPayloadPair.getFirst(); + List metadata = gatewayPayloadPair.getSecond(); + if (!CollectionUtils.isEmpty(metadata)) { + gatewayMetricsService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), metadata, systemTs); + } transportService.process(deviceCtx.getSessionInfo(), postTelemetryMsg, getPubAckCallback(channel, deviceName, msgId, postTelemetryMsg)); } catch (Throwable e) { log.warn("[{}][{}][{}] Failed to convert telemetry: [{}]", gateway.getTenantId(), gateway.getDeviceId(), deviceName, msg, e); @@ -394,25 +399,6 @@ public abstract class AbstractGatewaySessionHandler metadata) { - var serverReceiveTs = System.currentTimeMillis(); - var metricsData = metadata.stream() - .filter(JsonElement::isJsonObject) - .map(JsonElement::getAsJsonObject) - .filter(jo -> jo.has("connector") - && jo.has("receivedTs") - && jo.has("publishedTs")) - .map(jo -> { - var connector = jo.get("connector").getAsString(); - var receivedTs = jo.get("receivedTs").getAsLong(); - var publishedTs = jo.get("publishedTs").getAsLong(); - return new GatewayMetricsData(connector, receivedTs, publishedTs); - }).toList(); - if (!metricsData.isEmpty()) { - gatewayMetricsService.process(deviceSessionCtx.getSessionInfo(), gateway.getDeviceId(), metricsData, serverReceiveTs); - } - } - protected void onDeviceTelemetryProto(int msgId, ByteBuf payload) throws AdaptorException { try { TransportApiProtos.GatewayTelemetryMsg telemetryMsgProto = TransportApiProtos.GatewayTelemetryMsg.parseFrom(getBytes(payload)); From b886bfe4f700f9bf72faeaa2425385678634d682 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 13 Sep 2024 15:43:16 +0200 Subject: [PATCH 20/32] refactoring --- .../org/thingsboard/server/common/adaptor/JsonConverter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java index bffe59fa83..8dc021510f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java @@ -134,8 +134,7 @@ public class JsonConverter { } else if (jsonElement.isJsonArray()) { jsonElement.getAsJsonArray().forEach(je -> { if (je.isJsonObject()) { - JsonObject jo = je.getAsJsonObject(); - parseObject(systemTs, result, builder, jo); + parseObject(systemTs, result, builder, je.getAsJsonObject()); } else { throw new JsonSyntaxException(CAN_T_PARSE_VALUE + je); } From 118b2b7e10182e28e731b9232e64ab4b3ddc0a5d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Sep 2024 17:17:23 +0300 Subject: [PATCH 21/32] added upgrade script for device profile node --- .../service/install/SqlDatabaseUpgradeService.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 2d5ac5a99b..bfe389d6ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -122,7 +122,19 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService updateSchema("3.6.4", 3006004, "3.7.0", 3007000, null); break; case "3.7.0": - updateSchema("3.7.0", 3007000, "3.7.1", 3007001, null); + updateSchema("3.7.0", 3007000, "3.7.1", 3007001, connection -> { + try { + connection.createStatement().execute("UPDATE rule_node SET " + + "configuration = CASE " + + " WHEN (configuration::jsonb ->> 'persistAlarmRulesState')::boolean = false THEN jsonb_set(configuration::jsonb, '{fetchAlarmRulesStateOnStart}', 'false'::jsonb)::varchar " + + " ELSE configuration " + + "END, " + + "configuration_version = 1 " + + "WHERE type = 'org.thingsboard.rule.engine.profile.TbDeviceProfileNode' " + + "AND configuration_version < 1;"); + } catch (Exception e) { + } + }); break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); From 788b33007d8d6e7648c3aa6c3c963fb22d0976f9 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 13 Sep 2024 16:26:26 +0200 Subject: [PATCH 22/32] fixed NPE in RuleNOdeActorMessageProcessor --- .../actors/ruleChain/RuleNodeActorMessageProcessor.java | 7 ++++--- .../java/org/thingsboard/server/actors/TbActorMailbox.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index e7b0113dbe..46ffb6e894 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -43,6 +43,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; @Slf4j public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor { + private static final String UNKNOWN_NAME = "Unknown"; private final String ruleChainName; private final TbApiUsageReportClient apiUsageClient; private final DefaultTbContext defaultCtx; @@ -57,7 +58,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor msg.onTbActorStopped(stopReason)); normalPriorityMsgs.forEach(msg -> msg.onTbActorStopped(stopReason)); } catch (Throwable t) { - log.warn("[{}] Failed to destroy actor: {}", selfId, t); + log.warn("[{}] Failed to destroy actor: ", selfId, t); } }); } From 160b43e82c4842cd9245dc27e470dd11be80ecc3 Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 13 Sep 2024 22:26:26 +0300 Subject: [PATCH 23/32] lwm2m: fix bug Observe Composite - tests --- .../lwm2m/AbstractLwM2MIntegrationTest.java | 30 +- .../transport/lwm2m/Lwm2mTestHelper.java | 1 + .../client/LwM2mBinaryAppDataContainer.java | 19 +- .../lwm2m/client/SimpleLwM2MDevice.java | 7 +- ...bstractRpcLwM2MIntegrationObserveTest.java | 2 +- .../rpc/AbstractRpcLwM2MIntegrationTest.java | 12 +- ...cLwm2MIntegrationObserveCompositeTest.java | 466 ++++++++++-------- .../sql/RpcLwm2mIntegrationObserveTest.java | 163 +++--- .../DefaultLwM2mDownlinkMsgHandler.java | 201 ++++---- .../store/TbInMemoryRegistrationStore.java | 6 + .../uplink/DefaultLwM2mUplinkMsgHandler.java | 6 +- 11 files changed, 516 insertions(+), 397 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java index 983cfde0e5..af5be0dc77 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java @@ -411,27 +411,36 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte } protected void awaitObserveReadAll(int cntObserve, String deviceIdStr) throws Exception { - await("ObserveReadAll after start client/test: countObserve " + cntObserve) + await("ObserveReadAll: countObserve " + cntObserve) .atMost(40, TimeUnit.SECONDS) .until(() -> cntObserve == getCntObserveAll(deviceIdStr)); } protected Integer getCntObserveAll(String deviceIdStr) throws Exception { - String actualResultBefore = sendObserve("ObserveReadAll", null, deviceIdStr); - ObjectNode rpcActualResultBefore = JacksonUtil.fromString(actualResultBefore, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultBefore.get("result").asText()); - JsonElement element = JsonUtils.parse(rpcActualResultBefore.get("value").asText()); + String actualResult = sendObserveOK("ObserveReadAll", null, deviceIdStr); + ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + JsonElement element = JsonUtils.parse(rpcActualResult.get("value").asText()); return element.isJsonArray() ? ((JsonArray)element).size() : null; } - protected void sendCancelObserveAllWithAwait(String deviceIdStr) throws Exception { - String actualResultCancelAll = sendObserve("ObserveCancelAll", null, deviceIdStr); + protected void sendObserveCancelAllWithAwait(String deviceIdStr) throws Exception { + String actualResultCancelAll = sendObserveOK("ObserveCancelAll", null, deviceIdStr); ObjectNode rpcActualResultCancelAll = JacksonUtil.fromString(actualResultCancelAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultCancelAll.get("result").asText()); awaitObserveReadAll(0, deviceId); } - protected String sendObserve(String method, String params, String deviceIdStr) throws Exception { + protected String sendRpcObserveOkWithResultValue(String method, String params) throws Exception { + String actualResultReadAll = sendRpcObserveOk(method, params); + ObjectNode rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + return rpcActualResult.get("value").asText(); + } + protected String sendRpcObserveOk(String method, String params) throws Exception { + return sendObserveOK(method, params, deviceId); + } + protected String sendObserveOK(String method, String params, String deviceIdStr) throws Exception { String sendRpcRequest; if (params == null) { sendRpcRequest = "{\"method\": \"" + method + "\"}"; @@ -442,4 +451,9 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte return doPostAsync("/api/plugins/rpc/twoway/" + deviceIdStr, sendRpcRequest, String.class, status().isOk()); } + protected ObjectNode sendRpcObserveWithResult(String method, String params) throws Exception { + String actualResultReadAll = sendRpcObserveOk(method, params); + return JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java index b2ed3cc297..c627c77cd4 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/Lwm2mTestHelper.java @@ -48,6 +48,7 @@ public class Lwm2mTestHelper { public static final String RESOURCE_ID_NAME_3_9 = "batteryLevel"; public static final String RESOURCE_ID_NAME_3_14 = "UtfOffset"; public static final String RESOURCE_ID_NAME_19_0_0 = "dataRead"; + public static final String RESOURCE_ID_NAME_19_0_2 = "dataCreationTime"; public static final String RESOURCE_ID_NAME_19_1_0 = "dataWrite"; public static final String RESOURCE_ID_NAME_19_0_3 = "dataDescription"; diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2mBinaryAppDataContainer.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2mBinaryAppDataContainer.java index 88385a067e..3efd6365dd 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2mBinaryAppDataContainer.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/LwM2mBinaryAppDataContainer.java @@ -27,8 +27,10 @@ import org.eclipse.leshan.core.response.WriteResponse; import javax.security.auth.Destroyable; import java.sql.Time; +import java.time.Instant; +import java.time.LocalTime; +import java.time.ZoneId; import java.util.Arrays; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -85,7 +87,8 @@ public class LwM2mBinaryAppDataContainer extends BaseInstanceEnabler implements fireResourceChange(0); fireResourceChange(2); } - , 1800000, 1800000, TimeUnit.MILLISECONDS); // 30 MIN + , 1, 1, TimeUnit.SECONDS); // 1 sec +// , 1800000, 1800000, TimeUnit.MILLISECONDS); // 30 MIN } catch (Throwable e) { log.error("[{}]Throwable", e.toString()); e.printStackTrace(); @@ -123,6 +126,7 @@ public class LwM2mBinaryAppDataContainer extends BaseInstanceEnabler implements switch (resourceId) { case 0: if (setData(value, replace)) { + fireResourceChange(resourceId); return WriteResponse.success(); } else { WriteResponse.badRequest("Invalidate value ..."); @@ -132,7 +136,7 @@ public class LwM2mBinaryAppDataContainer extends BaseInstanceEnabler implements fireResourceChange(resourceId); return WriteResponse.success(); case 2: - setTimestamp(((Date) value.getValue()).getTime()); + setTimestamp(); fireResourceChange(resourceId); return WriteResponse.success(); case 3: @@ -177,12 +181,15 @@ public class LwM2mBinaryAppDataContainer extends BaseInstanceEnabler implements return this.description; } - private void setTimestamp(long time) { - this.timestamp = new Time(time); + private void setTimestamp() { + long currentTimeMillis = System.currentTimeMillis(); + this.timestamp = new Time(currentTimeMillis); } private Time getTimestamp() { - return this.timestamp != null ? this.timestamp : new Time(new Date().getTime()); + LocalTime localTime = LocalTime.ofInstant(Instant.now(), ZoneId.systemDefault()); + this.timestamp = Time.valueOf(localTime); + return this.timestamp; } private boolean setData(LwM2mResource value, boolean replace) { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SimpleLwM2MDevice.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SimpleLwM2MDevice.java index 1c0aa2b637..370c2249df 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SimpleLwM2MDevice.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SimpleLwM2MDevice.java @@ -58,7 +58,7 @@ public class SimpleLwM2MDevice extends BaseInstanceEnabler implements Destroyabl executorService.scheduleWithFixedDelay(() -> { fireResourceChange(9); } - , 1, 1, TimeUnit.SECONDS); // 30 MIN + , 1, 1, TimeUnit.SECONDS); // 2 sec // , 1800000, 1800000, TimeUnit.MILLISECONDS); // 30 MIN } catch (Throwable e) { log.error("[{}]Throwable", e.toString()); @@ -169,8 +169,9 @@ public class SimpleLwM2MDevice extends BaseInstanceEnabler implements Destroyabl } private int getBatteryLevel() { - return randomIterator.nextInt(); -// return 42; + int valBattery = randomIterator.nextInt(); + log.trace("Send from client [3/0/9] val: [{}]", valBattery); + return valBattery; } private long getMemoryFree() { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationObserveTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationObserveTest.java index 3cda275e13..ea9814c1d0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationObserveTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationObserveTest.java @@ -28,7 +28,7 @@ public abstract class AbstractRpcLwM2MIntegrationObserveTest extends AbstractRpc @Before public void initTest () throws Exception { - awaitObserveReadAll(2, deviceId); + awaitObserveReadAll(4, deviceId); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java index 1c5cf06c15..1f61a08e15 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/AbstractRpcLwM2MIntegrationTest.java @@ -42,8 +42,10 @@ import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INST import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_1; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_14; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_2; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_9; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_19_0_0; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_19_0_2; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_19_1_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_3_14; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_3_9; @@ -141,14 +143,18 @@ public abstract class AbstractRpcLwM2MIntegrationTest extends AbstractLwM2MInteg " \"" + idVer_3_0_9 + "\": \"" + RESOURCE_ID_NAME_3_9 + "\",\n" + " \"" + objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_14 + "\": \"" + RESOURCE_ID_NAME_3_14 + "\",\n" + " \"" + idVer_19_0_0 + "\": \"" + RESOURCE_ID_NAME_19_0_0 + "\",\n" + - " \"" + objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "\": \"" + RESOURCE_ID_NAME_19_1_0 + "\"\n" + + " \"" + objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "\": \"" + RESOURCE_ID_NAME_19_1_0 + "\",\n" + + " \"" + objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_2 + "\": \"" + RESOURCE_ID_NAME_19_0_2 + "\"\n" + " },\n" + " \"observe\": [\n" + " \"" + idVer_3_0_9 + "\",\n" + - " \"" + idVer_19_0_0 + "\"\n" + + " \"" + idVer_19_0_0 + "\",\n" + + " \"" + objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "\",\n" + + " \"" + objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_2 + "\"\n" + " ],\n" + " \"attribute\": [\n" + - " \"" + objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_14 + "\"\n" + + " \"" + objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_14 + "\",\n" + + " \"" + objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_2 + "\"\n" + " ],\n" + " \"telemetry\": [\n" + " \"" + idVer_3_0_9 + "\",\n" + diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java index 846d4091c0..cf78c58288 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2MIntegrationObserveCompositeTest.java @@ -15,10 +15,10 @@ */ package org.thingsboard.server.transport.lwm2m.rpc.sql; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.ResponseCode; -import org.eclipse.leshan.server.registration.Registration; import org.junit.Test; import org.mockito.Mockito; import org.springframework.boot.test.mock.mockito.SpyBean; @@ -26,24 +26,26 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.transport.lwm2m.rpc.AbstractRpcLwM2MIntegrationObserveTest; import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2mUplinkMsgHandler; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_1; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_14; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_15; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_2; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_3; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_5; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_7; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_9; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_19_0_0; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_19_0_2; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_19_1_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_3_14; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_NAME_3_9; @@ -63,7 +65,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt */ @Test public void testObserveCompositeAnyResources_Result_CONTENT_Value_LwM2mSingleResource_LwM2mResourceInstance() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; @@ -86,7 +88,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt */ @Test public void testObserveComposite_ObjectInstanceWithOtherObjectResourceInstance_Result_CONTENT_Ok() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; String expectedIdVer5_0 = objectInstanceIdVer_5; String expectedIds = "[\"" + expectedIdVer19_1_0 + "\", \"" + expectedIdVer5_0 + "\"]"; @@ -100,12 +102,12 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt /** * ObserveComposite {"ids":["5/0/7", "5/0/2"]} - Ok - * "5/0/2" - Execute^ result == null + * "5/0/2" - Execute result == null * @throws Exception */ @Test - public void testObserveCompositeAnyResources_Result_CONTENT_Value_LwM2mSingleResource_If_Error_Null() throws Exception { -// sendCancelObserveAllWithAwait(deviceId); + public void testObserveReadAll_AfterCompositeObservation_WithResourceNotReadable_Result_CONTENT_ObserveResourceNotReadableIsNull() throws Exception { + sendObserveCancelAllWithAwait(deviceId); String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_2 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_2 + "\"]"; @@ -125,7 +127,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt */ @Test public void testObserveComposite_Result_BAD_REQUEST_ONE_PATH_CONTAINCE_OTHER() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String expectedIdVer5_0 = objectInstanceIdVer_5; String expectedIdVer5_0_2 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; String expectedIds = "[\"" + expectedIdVer5_0 + "\", \"" + expectedIdVer5_0_2 + "\"]"; @@ -138,21 +140,89 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt } /** - * Previous -> "3/0/9" - * ObserveComposite {"ids":["5/0/7", "5/0/5", "5/0/3", "3/0/9"]} - CONTENT + * Previous -> "3/0/9", "19/0/2", "19/1/0", "19/0/0", All only SingleObservation; + * if at least one of the resource objectIds (Composite) in SingleObservation or CompositeObservation is already registered - return BAD REQUEST + * ObserveComposite {"ids":["5/0/7", "5/0/5", "5/0/3", "3/0/9"]} * @throws Exception */ @Test - public void testObserveCompositeThereAreObservationOneResource_Result_CONTENT_Value_ObservationAddIfAbsent() throws Exception { + public void testObserveComposite_IfLeastOneResourceIsAlreadyRegistered_return_BadRequest() throws Exception { + // Verify after start + String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); + ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); + String actualValues = rpcActualResultReadAll.get("value").asText(); + String expectedIdVer19_0_2 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_2; + String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_9))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_0_2))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_1_0))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_19_0_0))); + // Send Observe composite with "/3/0/9" String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + idVer_3_0_9 + "\"]"; String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + assertTrue(rpcActualResult.get("error").asText().contains(fromVersionedIdToObjectId(idVer_3_0_9))); + assertTrue(rpcActualResult.get("error").asText().contains("is already registered")); + // verify after send Observe composite + actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); + rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); + actualValues = rpcActualResultReadAll.get("value").asText(); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_9))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_0_2))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_1_0))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_19_0_0))); + } + /** + * Previous -> ["5/0/7", "5/0/5", "5/0/3"], CompositeObservation * + * if the resource SingleObservation is already registered in CompositeObservation - return BAD REQUEST + * SingleObservation {"id":["5/0/7"} + * @throws Exception + */ + @Test + public void testObserveSingle_IfResourceIsAlreadyRegisteredInComposite_return_BadRequest() throws Exception { + // Send Observe Composite + String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; + String expectedId5_0_7 = fromVersionedIdToObjectId(expectedIdVer5_0_7); + String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; + String expectedId5_0_5 = fromVersionedIdToObjectId(expectedIdVer5_0_5); + String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; + String expectedId5_0_3 = fromVersionedIdToObjectId(expectedIdVer5_0_3); + + String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_5 + "\", \"" + expectedIdVer5_0_3 + "\"]"; + String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); + ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String expectedResult = "/3/0/9=LwM2mSingleResource [id=9"; - assertTrue(rpcActualResult.get("value").asText().contains(expectedResult)); + String actualValues = rpcActualResult.get("value").asText(); + assertTrue(actualValues.contains(expectedId5_0_7)); + assertTrue(actualValues.contains(expectedId5_0_5)); + assertTrue(actualValues.contains(expectedId5_0_3)); + // Send Observe Single + actualResult = sendObserve("Observe", expectedIdVer5_0_7); + rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + assertTrue(rpcActualResult.get("error").asText().contains(expectedId5_0_7)); + assertTrue(rpcActualResult.get("error").asText().contains("is already registered")); + // verify after send Observe Single + String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); + ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); + actualValues = rpcActualResultReadAll.get("value").asText(); + String expectedIdVer19_0_2 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_2; + String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_9))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_0_2))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_1_0))); + assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_19_0_0))); + assertTrue(actualValues.contains("CompositeObservation:")); + assertTrue(actualValues.contains(expectedId5_0_7)); + assertTrue(actualValues.contains(expectedId5_0_5)); + assertTrue(actualValues.contains(expectedId5_0_3)); } /** @@ -161,7 +231,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt */ @Test public void testObserveCompositeAnyResources_Result_CONTENT_Value_LwM2mSingleResource_LwM2mMultipleResource() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; @@ -185,7 +255,7 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt */ @Test public void testObserveCompositeWithKeyName_Result_CONTENT_Value_SingleResources() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String expectedKey3_0_9 = RESOURCE_ID_NAME_3_9; String expectedKey3_0_14 = RESOURCE_ID_NAME_3_14; @@ -210,18 +280,16 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt * @throws Exception */ @Test - public void testObserveCompositeWithKeyNameThereAreObservationOneResource_Result_CONTENT_Value_ObservationAddIfAbsent() throws Exception { + public void testObserveCompositeWithKeyName_IfLeastOneResourceIsAlreadyRegistered_return_BadRequest() throws Exception { String expectedKey3_0_9 = RESOURCE_ID_NAME_3_9; String expectedKey3_0_14 = RESOURCE_ID_NAME_3_14; String expectedKey19_0_0 = RESOURCE_ID_NAME_19_0_0; String expectedKey19_1_0 = RESOURCE_ID_NAME_19_1_0; String expectedKeys = "[\"" + expectedKey3_0_9 + "\", \"" + expectedKey3_0_14 + "\", \"" + expectedKey19_0_0 + "\", \"" + expectedKey19_1_0 + "\"]"; - String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; String actualResult = sendCompositeRPCByKeys("ObserveComposite", expectedKeys); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actual = rpcActualResult.get("value").asText(); - assertTrue(actual.contains(fromVersionedIdToObjectId(expectedIdVer19_1_0) + "=LwM2mMultipleResource")); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + assertTrue(rpcActualResult.get("error").asText().contains("is already registered")); } /** @@ -230,8 +298,8 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt * @throws Exception */ @Test - public void testObserveReadAll_AfterCompositeObservation_Result_CONTENT_Value_SingleObservation_Only() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + public void testObserveReadAll_AfterbserveCancelAllAndCompositeObservation_Result_CONTENT_Value_CompositeObservation_Only() throws Exception { + sendObserveCancelAllWithAwait(deviceId); String expectedKey3_0_9 = RESOURCE_ID_NAME_3_9; String expectedKey3_0_14 = RESOURCE_ID_NAME_3_14; @@ -247,72 +315,12 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualValues = rpcActualResultReadAll.get("value").asText(); String expectedIdVer3_0_14 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_14; String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_9))); - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer3_0_14))); - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_1_0))); - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_19_0_0))); - } - - /** - * ObserveReadAll - * {"result":"CONTENT","value":"{"result":"CONTENT","value":"["SingleObservation:/3/0/9","SingleObservation:/3/0/14","SingleObservation:/19/1/0/0","SingleObservation:/19/0/0"]"} - Ok - * @throws Exception - */ - @Test - public void testObserveReadAll_Result_CONTENT_Value_SingleObservation_Only() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - - String expectedIdVer3_0_14 = objectInstanceIdVer_3 + "/" + RESOURCE_ID_14; - String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; - String actualResult3_0_9 = sendObserve("Observe", idVer_3_0_9); - ObjectNode rpcActualResult3_0_9 = JacksonUtil.fromString(actualResult3_0_9, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult3_0_9.get("result").asText()); - String actualResult3_0_14 = sendObserve("Observe", expectedIdVer3_0_14); - ObjectNode rpcActualResult3_0_14 = JacksonUtil.fromString(actualResult3_0_14, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult3_0_14.get("result").asText()); - String actualResult19_1_0_0 = sendObserve("Observe", expectedIdVer19_1_0_0); - ObjectNode rpcActualResult19_1_0_0 = JacksonUtil.fromString(actualResult19_1_0_0, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult19_1_0_0.get("result").asText()); - String actualResult19_0_0 = sendObserve("Observe", idVer_19_0_0); - ObjectNode rpcActualResult19_0_0 = JacksonUtil.fromString(actualResult19_0_0, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult19_0_0.get("result").asText()); - String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); - ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_9))); - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer3_0_14))); - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer19_1_0_0))); - assertTrue(actualValues.contains("SingleObservation:" + fromVersionedIdToObjectId(idVer_19_0_0))); - } - - /** - * ObserveReadAll - * {"result":"CONTENT","value":"[\"CompositeObservation: [/19/1/0\",\"/19/0/0\",\"/3/0/14\",\"/3/0/9]\"]"} - Ok - * @throws Exception - */ - @Test - public void testObserveReadAll_AfterCompositeObservation_WithResourceNotReadable_Result_CONTENT_Value_SingleObservation_Only() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - - String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; - String expectedIdVer5_0_2 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_2; - String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; - String expectedIdVer19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; - String expectedIds = "[\"" + expectedIdVer5_0_7 + "\", \"" + expectedIdVer5_0_2 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + idVer_3_0_9 + "\", \"" + expectedIdVer19_1_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - String actualValues = rpcActualResult.get("value").asText(); - - assertTrue(actualValues.contains(fromVersionedIdToObjectId(expectedIdVer5_0_2) + "=null")); - - String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); - ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - actualValues = rpcActualResultReadAll.get("value").asText(); - - assertFalse(actualValues.contains(fromVersionedIdToObjectId(expectedIdVer5_0_2))); + assertTrue(actualValues.contains("CompositeObservation:")); + assertFalse(actualValues.contains("SingleObservation")); + assertTrue(actualValues.contains(Objects.requireNonNull(fromVersionedIdToObjectId(idVer_3_0_9)))); + assertTrue(actualValues.contains(Objects.requireNonNull(fromVersionedIdToObjectId(expectedIdVer3_0_14)))); + assertTrue(actualValues.contains(Objects.requireNonNull(fromVersionedIdToObjectId(expectedIdVer19_1_0)))); + assertTrue(actualValues.contains(Objects.requireNonNull(fromVersionedIdToObjectId(idVer_19_0_0)))); } /** @@ -321,8 +329,8 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt * @throws Exception */ @Test - public void testObserveCompositeAnyResources_Result_CONTENT_CancelObserveComposite_This_Result_Content_Count_5() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + public void testObserveCancelAllThenObserveCompositeAnyResources_Result_CONTENT_CancelObserveComposite_This_Result_Content_Count_1() throws Exception { + sendObserveCancelAllWithAwait(deviceId); // ObserveComposite String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; @@ -336,45 +344,20 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("5", rpcActualResult.get("value").asText()); - - assertEquals(0, (Object) getCntObserveAll(deviceId)); - } - - /** - * ObserveComposite {"ids":["/3", "/5/0/3", "/19/1/0/0"]} - Ok - * ObserveCompositeCancel {"ids":["/3", "/5/0/3", "/19/1/0/0"]} - Ok - * @throws Exception - */ - @Test - public void testObserveCompositeOneObjectAnyResources_Result_CONTENT_CancelObserveComposite_This_Result_Content_Count_3() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite - String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; - String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; - String expectedIds = "[\"" + idVer_3_0_9 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // ObserveCompositeCancel - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("3", rpcActualResult.get("value").asText()); + assertEquals("1", rpcActualResult.get("value").asText()); assertEquals(0, (Object) getCntObserveAll(deviceId)); } /** * ObserveComposite {"ids":["/3/0/9", "/5/0/5", "/5/0/3", "/5/0/7", "/19/1/0/0"]} - Ok - * ObserveCompositeCancel {"ids":["/5", "/19/1/0/0"]} - Ok - * last Observation + * ObserveCompositeCancel {"ids":["/5", "/19/1/0/0"]} - BadRequest * @throws Exception */ @Test - public void testObserveCompositeAnyResources_Result_CONTENT_CancelObserveComposite_OneObjectAnyResource_Result_Content_Count_4() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite + public void testObserveCompositeFiveResources_Result_CONTENT_CancelObserveComposite_TwoAnyResource_Result_BadRequest() throws Exception { + sendObserveCancelAllWithAwait(deviceId); + // ObserveComposite five String expectedIdVer5_0_7 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_7; String expectedIdVer5_0_5 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_5; String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; @@ -384,32 +367,25 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - awaitObserveReadAll(5, deviceId); + awaitObserveReadAll(1, deviceId); - // ObserveCompositeCancel + // ObserveCompositeCancel two expectedIds = "[\"" + objectInstanceIdVer_5 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("4", rpcActualResult.get("value").asText()); // CNT = 4 ("/5/0/5", "/5/0/3", "/5/0/7", "/19/1/0/0"9) - - String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); - ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); - assertEquals("[\"SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer3_0_9) + "\"]", actualValues); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + assertTrue(rpcActualResult.get("error").asText().contains(objectInstanceIdVer_5)); // CNT = 4 ("/5/0/5", "/5/0/3", "/5/0/7", "/19/1/0/0"9) + assertTrue(rpcActualResult.get("error").asText().contains(expectedIdVer19_1_0_0)); // CNT = 4 ("/5/0/5", "/5/0/3", "/5/0/7", "/19/1/0/0"9) } /** * ObserveComposite {"ids":["/3", "/5/0/3", "/19/1/0/0"]} - Ok - * ObserveCompositeCancel {"ids":["/3/0/9", "/5/0/3", "/19/1/0/0"} -> BAD_REQUEST - * ObserveCompositeCancel {"ids":["/3"} -> CONTENT + * ObserveCompositeCancel {"ids":["/19/1/0/0", "/3/0/9"} -> BAD_REQUEST */ @Test public void testObserveOneObjectAnyResources_Result_CONTENT_Cancel_OneResourceFromObjectAnyResource_Result_BAD_REQUEST_Cancel_OneObject_Result_CONTENT() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); // ObserveComposite - sendCancelObserveAllWithAwait(deviceId); String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; String expectedIds = "[\"" + objectIdVer_3 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; @@ -418,89 +394,107 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); // ObserveCompositeCancel - expectedIds = "[\"" + expectedIdVer19_1_0_0 + "\", \"" + idVer_3_0_9 + "\"]"; - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); + String sendIds = "[\"" + expectedIdVer19_1_0_0 + "\", \"" + idVer_3_0_9 + "\"]"; + actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", sendIds); rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); - String expectedValue = "for observation path [" + fromVersionedIdToObjectId(objectIdVer_3) + "], that includes this observation path [" + fromVersionedIdToObjectId(idVer_3_0_9); + String expectedValue = "Could not find active Observe Composite component with paths: [/19_1.1/1/0/0, /3_1.2/0/9]"; assertTrue(rpcActualResult.get("error").asText().contains(expectedValue)); - // ObserveCompositeCancel - expectedIds = "[\"" + objectIdVer_3 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("2", rpcActualResult.get("value").asText()); - + // "ObserveReadAll" String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); String actualValues = rpcActualResultReadAll.get("value").asText(); - assertEquals("[\"SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer5_0_3) + "\"]", actualValues); + assertTrue(actualValues.contains("CompositeObservation:")); } - /** - * ObserveComposite {"ids":["/3/0/9", "/3/0/14", "/5/0/3", "/3/0/15", "/19/1/0/0"]} - Ok - * ObserveCancel {"id":"/3/0/9"} -> INTERNAL_SERVER_ERROR - * ObserveCompositeCancel {"ids":["/3/0/9", "/19/1/0/0", "/3]} - Ok - * last Observation - * @throws Exception - */ @Test - public void testObserveCompositeAnyResources_Result_CONTENT_CancelObserveComposite_OneResource_OneObjectAnyResource_Result_Content_Count_4() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - // ObserveComposite - sendCancelObserveAllWithAwait(deviceId); - String expectedIdVer3_0_14 = objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_14; - String expectedIdVer3_0_15 = objectIdVer_3 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_15; - String expectedIdVer5_0_3 = objectInstanceIdVer_5 + "/" + RESOURCE_ID_3; - String expectedIdVer19_1_0_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0 + "/" + RESOURCE_INSTANCE_ID_0; - String expectedIds = "[\"" + idVer_3_0_9 + "\", \"" + expectedIdVer3_0_14 + "\", \"" + expectedIdVer5_0_3 + "\", \"" + expectedIdVer3_0_15 + "\", \"" + expectedIdVer19_1_0_0 + "\"]"; - String actualResult = sendCompositeRPCByIds("ObserveComposite", expectedIds); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - // ObserveCompositeCancel - expectedIds = "[\"" + idVer_3_0_9 + "\", \"" + expectedIdVer19_1_0_0 + "\", \"" + objectIdVer_3 + "\"]"; - actualResult = sendCompositeRPCByIds("ObserveCompositeCancel", expectedIds); - rpcActualResult = JacksonUtil.fromString(actualResult, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - assertEquals("4", rpcActualResult.get("value").asText()); - + public void testObserveCompositeResource_Update_After_Registration_UpdateRegistration() throws Exception { + String id_3_0_9 = fromVersionedIdToObjectId(idVer_3_0_9); + String id_19_0_0 = fromVersionedIdToObjectId(idVer_19_0_0); + String idVer_19_1_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1 + "/" + RESOURCE_ID_0; + String id_19_1_0 = fromVersionedIdToObjectId(idVer_19_1_0); + String idVer_19_0_2 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0 + "/" + RESOURCE_ID_2; + String id_19_0_2 = fromVersionedIdToObjectId(idVer_19_0_2); + + // 1 - "ObserveReadAll": at least one update value of all resources we observe - after connection String actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); ObjectNode rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); - String actualValues = rpcActualResultReadAll.get("value").asText(); - assertEquals("[\"SingleObservation:" + fromVersionedIdToObjectId(expectedIdVer5_0_3) + "\"]", actualValues); - } - - /** - * ObserveCancelAll - * Observe {"id":"/3/0/9"} - * updateRegistration - * idResources_/3/0/9 => updateAttrTelemetry >= 10 times - */ - @Test - public void testObserveCompositeResource_Update_AfterUpdateRegistration() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - - int cntUpdate = 3; - verify(defaultUplinkMsgHandlerTest, timeout(50000).atLeast(cntUpdate)) - .updatedReg(Mockito.any(Registration.class)); - - log.warn("After ObserveReadAll after cancel - send composite observe /3303_1.0/0/5700"); - - String expectedKey3_0_9 = RESOURCE_ID_NAME_3_9; - String expectedKey3_0_14 = RESOURCE_ID_NAME_3_14; - String expectedKey19_0_0 = RESOURCE_ID_NAME_19_0_0; - String expectedKey19_1_0 = RESOURCE_ID_NAME_19_1_0; - String expectedKeys = "[\"" + expectedKey3_0_9 + "\", \"" + expectedKey3_0_14 + "\", \"" + expectedKey19_0_0 + "\", \"" + expectedKey19_1_0 + "\", \"" + expectedKey3_0_9 + "\"]"; + String rpcActualVValuesReadAll = rpcActualResultReadAll.get("value").asText(); + ArrayNode rpcactualValues = JacksonUtil.fromString(rpcActualVValuesReadAll, ArrayNode.class); + assertEquals(rpcactualValues.size(), 4); + assertTrue(actualResultReadAll.contains("SingleObservation:" + id_3_0_9)); + assertTrue(actualResultReadAll.contains("SingleObservation:" + id_19_1_0)); + assertTrue(actualResultReadAll.contains("SingleObservation:" + id_19_0_2)); + assertTrue(actualResultReadAll.contains("SingleObservation:" + id_19_0_0)); + long initAttrTelemetryAtCount = countUpdateAttrTelemetryAll(); + long initAttrTelemetryAtCount_3_0_9 = countUpdateAttrTelemetryResource(idVer_3_0_9); + long initAttrTelemetryAtCount_19_0_0 = countUpdateAttrTelemetryResource(idVer_19_0_0); + long initAttrTelemetryAtCount_19_1_0 = countUpdateAttrTelemetryResource(idVer_19_1_0); + long initAttrTelemetryAtCount_19_0_2 = countUpdateAttrTelemetryResource(idVer_19_0_2); + updateRegAtLeastOnceAfterAction(); + updateAttrTelemetryAllAtLeastOnceAfterAction(initAttrTelemetryAtCount); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_3_0_9, idVer_3_0_9); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_19_0_0, idVer_19_0_0); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_19_1_0, idVer_19_1_0); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_19_0_2, idVer_19_0_2); + + // 2 - "ObserveReadAll": No update of all resources we are observing - after "ObserveReadCancelAll" + sendObserveCancelAllWithAwait(deviceId); + updateRegAtLeastOnceAfterAction(); + actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); + rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); + rpcActualVValuesReadAll = rpcActualResultReadAll.get("value").asText(); + rpcactualValues = JacksonUtil.fromString(rpcActualVValuesReadAll, ArrayNode.class); + assertEquals(rpcactualValues.size(), 0); + // 2.1 - ObserveComposite: observeCancelAll verify" + initAttrTelemetryAtCount = countUpdateAttrTelemetryAll(); + initAttrTelemetryAtCount_3_0_9 = countUpdateAttrTelemetryResource(idVer_3_0_9); + initAttrTelemetryAtCount_19_0_0 = countUpdateAttrTelemetryResource(idVer_19_0_0); + initAttrTelemetryAtCount_19_1_0 = countUpdateAttrTelemetryResource(idVer_19_1_0); + initAttrTelemetryAtCount_19_0_2 = countUpdateAttrTelemetryResource(idVer_19_0_2); + updateRegAtLeastOnceAfterAction(); + assertEquals(countUpdateAttrTelemetryAll(), initAttrTelemetryAtCount); + assertEquals(countUpdateAttrTelemetryResource(idVer_3_0_9), initAttrTelemetryAtCount_3_0_9); + assertEquals(countUpdateAttrTelemetryResource(idVer_19_0_0), initAttrTelemetryAtCount_19_0_0); + assertEquals(countUpdateAttrTelemetryResource(idVer_19_1_0), initAttrTelemetryAtCount_19_1_0); + assertEquals(countUpdateAttrTelemetryResource(idVer_19_0_2), initAttrTelemetryAtCount_19_0_2); + + // 3 - ObserveComposite: at least one update value of all resources we observe - after ObserveComposite" + String expectedKeys = "[\"" + RESOURCE_ID_NAME_3_9 + "\", \"" + RESOURCE_ID_NAME_19_0_0 + "\", \"" + RESOURCE_ID_NAME_19_0_2 + "\", \"" + RESOURCE_ID_NAME_19_1_0 + "\"]"; String actualResult = sendCompositeRPCByKeys("ObserveComposite", expectedKeys); - - - cntUpdate = 20; - verify(defaultUplinkMsgHandlerTest, timeout(50000).atLeast(cntUpdate)) - .updateAttrTelemetry(Mockito.any(Registration.class), argThat(arg -> arg.equals(idVer_3_0_9) || arg.equals(idVer_19_0_0))); + assertTrue(actualResult.contains(id_3_0_9 + "=LwM2mSingleResource")); + assertTrue(actualResult.contains(id_19_0_0 + "=LwM2mMultipleResource")); + assertTrue(actualResult.contains(id_19_1_0 + "=LwM2mMultipleResource")); + assertTrue(actualResult.contains(id_19_0_2 + "=LwM2mSingleResource")); + // 3.1 - ObserveComposite: - verify"); + actualResultReadAll = sendCompositeRPCByKeys("ObserveReadAll", null); + rpcActualResultReadAll = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultReadAll.get("result").asText()); + rpcActualVValuesReadAll = rpcActualResultReadAll.get("value").asText(); + rpcactualValues = JacksonUtil.fromString(rpcActualVValuesReadAll, ArrayNode.class); + assertEquals(rpcactualValues.size(), 1); + assertFalse(actualResultReadAll.contains("SingleObservation")); + assertTrue(actualResultReadAll.contains("CompositeObservation:")); + assertTrue(actualResultReadAll.contains(id_19_0_2)); + assertTrue(actualResultReadAll.contains(id_19_1_0)); + assertTrue(actualResultReadAll.contains(id_19_0_0)); + assertTrue(actualResultReadAll.contains(id_3_0_9)); + initAttrTelemetryAtCount = countUpdateAttrTelemetryAll(); + initAttrTelemetryAtCount_3_0_9 = countUpdateAttrTelemetryResource(idVer_3_0_9); + initAttrTelemetryAtCount_19_0_0 = countUpdateAttrTelemetryResource(idVer_19_0_0); + initAttrTelemetryAtCount_19_1_0 = countUpdateAttrTelemetryResource(idVer_19_1_0); + initAttrTelemetryAtCount_19_0_2 = countUpdateAttrTelemetryResource(idVer_19_0_2); + updateRegAtLeastOnceAfterAction(); + updateAttrTelemetryAllAtLeastOnceAfterAction(initAttrTelemetryAtCount); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_3_0_9, idVer_3_0_9); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_19_0_0, idVer_19_0_0); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_19_1_0, idVer_19_1_0); + updateAttrTelemetryResourceAtLeastOnceAfterAction(initAttrTelemetryAtCount_19_0_2, idVer_19_0_2); } private String sendObserve(String method, String params) throws Exception { @@ -522,4 +516,68 @@ public class RpcLwm2MIntegrationObserveCompositeTest extends AbstractRpcLwM2MInt String sendRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"keys\":" + keys + "}}"; return doPostAsync("/api/plugins/rpc/twoway/" + deviceId, sendRpcRequest, String.class, status().isOk()); } + + private long countUpdateAttrTelemetryAll() { + return Mockito.mockingDetails(defaultUplinkMsgHandlerTest) + .getInvocations().stream() + .filter(invocation -> invocation.getMethod().getName().equals("updateAttrTelemetry")) + .count(); + } + + private void updateAttrTelemetryAllAtLeastOnceAfterAction(long initialInvocationCount) { + AtomicLong newInvocationCount = new AtomicLong(initialInvocationCount); + log.warn("countUpdateAttrTelemetryAllAtLeastOnceAfterAction: initialInvocationCount [{}]", initialInvocationCount); + await("Update AttrTelemetryAll at-least-once after action") + .atMost(50, TimeUnit.SECONDS) + .until(() -> { + newInvocationCount.set(countUpdateAttrTelemetryAll()); + return newInvocationCount.get() > initialInvocationCount; + }); + log.warn("countUpdateAttrTelemetryAllAtLeastOnceAfterAction: newInvocationCount [{}]", newInvocationCount.get()); + } + + + private long countUpdateAttrTelemetryResource(String idVerRez) { + return Mockito.mockingDetails(defaultUplinkMsgHandlerTest) + .getInvocations().stream() + .filter(invocation -> + invocation.getMethod().getName().equals("updateAttrTelemetry") && + invocation.getArguments().length > 1 && + idVerRez.equals(invocation.getArguments()[1]) + ) + .count(); + } + + + private void updateAttrTelemetryResourceAtLeastOnceAfterAction(long initialInvocationCount, String idVerRez) { + AtomicLong newInvocationCount = new AtomicLong(initialInvocationCount); + log.warn("countUpdateAttrTelemetryResourceAtLeastOnceAfterAction: initialInvocationCount [{}]", initialInvocationCount); + await("Update AttrTelemetryResource at-least-once after action") + .atMost(50, TimeUnit.SECONDS) + .until(() -> { + newInvocationCount.set(countUpdateAttrTelemetryResource(idVerRez)); + return newInvocationCount.get() > initialInvocationCount; + }); + log.warn("countUpdateAttrTelemetryResourceAtLeastOnceAfterAction: newInvocationCount [{}]", newInvocationCount.get()); + } + + private long countUpdateReg() { + return Mockito.mockingDetails(defaultUplinkMsgHandlerTest) + .getInvocations().stream() + .filter(invocation -> invocation.getMethod().getName().equals("updatedReg")) + .count(); + } + + private void updateRegAtLeastOnceAfterAction() { + long initialInvocationCount = countUpdateReg(); + AtomicLong newInvocationCount = new AtomicLong(initialInvocationCount); + log.warn("updateRegAtLeastOnceAfterAction: initialInvocationCount [{}]", initialInvocationCount); + await("Update Registration at-least-once after action") + .atMost(50, TimeUnit.SECONDS) + .until(() -> { + newInvocationCount.set(countUpdateReg()); + return newInvocationCount.get() > initialInvocationCount; + }); + log.warn("updateRegAtLeastOnceAfterAction: newInvocationCount [{}]", newInvocationCount.get()); + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java index 8fdf1994b6..ce2622c4db 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationObserveTest.java @@ -25,7 +25,6 @@ import org.eclipse.leshan.server.registration.Registration; import org.junit.Test; import org.mockito.Mockito; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.transport.lwm2m.rpc.AbstractRpcLwM2MIntegrationObserveTest; import org.thingsboard.server.transport.lwm2m.server.uplink.DefaultLwM2mUplinkMsgHandler; @@ -38,10 +37,10 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_0; +import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.OBJECT_INSTANCE_ID_1; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_0; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_2; import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_3; -import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID_9; import static org.thingsboard.server.transport.lwm2m.utils.LwM2MTransportUtil.fromVersionedIdToObjectId; @Slf4j @@ -51,13 +50,12 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO DefaultLwM2mUplinkMsgHandler defaultUplinkMsgHandlerTest; @Test - public void testObserveReadAll_Count_2_CancelAll_Count_0_Ok() throws Exception { - String actualValuesReadAll = sendRpcObserveWithResultValue("ObserveReadAll", null); - assertEquals(2, actualValuesReadAll.split(",").length); - String expected = "\"SingleObservation:/19/0/0\""; - assertTrue(actualValuesReadAll.contains(expected)); - expected = "\"SingleObservation:/3/0/9\""; - assertTrue(actualValuesReadAll.contains(expected)); + public void testObserveReadAll_Count_4_CancelAll_Count_0_Ok() throws Exception { + String actualValuesReadAll = sendRpcObserveOkWithResultValue("ObserveReadAll", null); + assertEquals(4, actualValuesReadAll.split(",").length); + sendObserveCancelAllWithAwait(deviceId); + actualValuesReadAll = sendRpcObserveOkWithResultValue("ObserveReadAll", null); + assertEquals("[]", actualValuesReadAll); } /** @@ -66,7 +64,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveOneResource_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0_9); int cntUpdate = 3; @@ -80,7 +78,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveOneObjectInstance_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String idVer_3_0 = objectInstanceIdVer_3; sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0); @@ -95,7 +93,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveOneObject_Result_CONTENT_Value_Count_3_After_Cancel_Count_2() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String idVer_3_0 = objectInstanceIdVer_3; sendRpcObserveWithContainsLwM2mSingleResource(idVer_3_0); @@ -111,8 +109,8 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveRepeated_Result_CONTENT_AddIfAbsent() throws Exception { - sendRpcObserveWithResultValue("Observe", idVer_3_0_0); - String rpcActualResult = sendRpcObserveWithResultValue("Observe", idVer_3_0_0); + sendRpcObserveOkWithResultValue("Observe", idVer_3_0_0); + String rpcActualResult = sendRpcObserveOkWithResultValue("Observe", idVer_3_0_0); String expected = "LwM2mSingleResource [id=0"; assertTrue(rpcActualResult.contains(expected)); } @@ -190,48 +188,61 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveRepeatedRequestObserveOnDevice_Result_CONTENT_PutIfAbsent() throws Exception { - sendRpcObserveWithResultValue("Observe", idVer_3_0_0); - String rpcActualResult = sendRpcObserveWithResultValue("Observe", idVer_3_0_0); + sendRpcObserveOkWithResultValue("Observe", idVer_3_0_0); + String rpcActualResult = sendRpcObserveOkWithResultValue("Observe", idVer_3_0_0); String expected = "LwM2mSingleResource [id=0"; assertTrue(rpcActualResult.contains(expected)); } /** - * Observe {"id":["3"]} - Ok * PreviousObservation contains "3/0/9" + * Observe {"id":["19"]} - Bad Request + * Observe {"id":["19/0"]} - Bad Request + * Observe {"id":["19/1"]} - Ok * @throws Exception */ @Test - public void testObserve_Result_CONTENT_ONE_PATH_PreviousObservation_CONTAINCE_OTHER_CurrentObservation() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - // "3/0/9" - sendRpcObserveWithResultValue("Observe", idVer_3_0_9); - // "3" - sendRpcObserveWithResultValue("Observe", objectIdVer_3); - // PreviousObservation "3/0/9" change to CurrentObservation "3" - String actualValuesReadAll = sendRpcObserveWithResultValue("ObserveReadAll", null); - assertEquals(1, actualValuesReadAll.split(",").length); - String expected = "\"SingleObservation:/3\""; - assertTrue(actualValuesReadAll.contains(expected)); - } - - /** - * Observe {"id":["3/0/9"]} - Ok - * PreviousObservation contains "3" - * @throws Exception - */ - @Test - public void testObserve_Result_CONTENT_ONE_PATH_CurrentObservation_CONTAINCE_OTHER_PreviousObservation() throws Exception { - sendCancelObserveAllWithAwait(deviceId); - // "3" - sendRpcObserveWithResultValue("Observe", objectIdVer_3); - // "3/0/0"; WARN: - Token collision ? existing observation [/3] includes input observation [/3/0/0] - sendRpcObserveWithResultValue("Observe", idVer_3_0_0); - - String actualValuesReadAll = sendRpcObserveWithResultValue("ObserveReadAll", null); - assertEquals(1, actualValuesReadAll.split(",").length); - String expected = "\"SingleObservation:/3\""; - assertTrue(actualValuesReadAll.contains(expected)); + public void testObserves_OverlappedPaths_FirstResource_SecondObjectOrInstance() throws Exception { + sendObserveCancelAllWithAwait(deviceId); + // "19/0/0" + sendRpcObserveOkWithResultValue("Observe", idVer_19_0_0); + // PreviousObservation "19/0/0" change to CurrentObservation "19" - object + ObjectNode rpcActualResult = sendRpcObserveWithResult("Observe", objectIdVer_19); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + String expected = "Resource [" + fromVersionedIdToObjectId(objectIdVer_19) + "] conflict with is already registered as SingleObservation [" + fromVersionedIdToObjectId(idVer_19_0_0) + "]."; + assertEquals(expected, rpcActualResult.get("error").asText()); + // Verify ObserveReadAll + String actualValuesReadAll = sendRpcObserveOkWithResultValue("ObserveReadAll", null); + String expectedReadAll = "[\"SingleObservation:/19/0/0\"]"; + assertEquals(expectedReadAll, actualValuesReadAll); + // PreviousObservation "19/0/0" change to CurrentObservation "19/0" - instance + String expectedIdVer19_0 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_0; + rpcActualResult = sendRpcObserveWithResult("Observe", expectedIdVer19_0); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + expected = "Resource [" + fromVersionedIdToObjectId(expectedIdVer19_0) + "] conflict with is already registered as SingleObservation [" + fromVersionedIdToObjectId(idVer_19_0_0) + "]."; + assertEquals(expected, rpcActualResult.get("error").asText()); + // Verify ObserveReadAll + actualValuesReadAll = sendRpcObserveOkWithResultValue("ObserveReadAll", null); + assertEquals(expectedReadAll, actualValuesReadAll); + // PreviousObservation "19/0/0" add CurrentObservation "19/1" - instance + String expectedIdVer19_1 = objectIdVer_19 + "/" + OBJECT_INSTANCE_ID_1; + rpcActualResult = sendRpcObserveWithResult("Observe", expectedIdVer19_1); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + assertTrue(rpcActualResult.get("value").asText().contains("LwM2mObjectInstance")); + // Verify ObserveReadAll + actualValuesReadAll = sendRpcObserveOkWithResultValue("ObserveReadAll", null); + assertTrue(actualValuesReadAll.contains("SingleObservation:/19/1")); + assertTrue(actualValuesReadAll.contains("SingleObservation:/19/0/0")); + // PreviousObservation "19/1/"- instance change to CurrentObservation "19/1/0" - resource + String expectedIdVer19_1_0 = expectedIdVer19_1 + "/" + RESOURCE_ID_0; + rpcActualResult = sendRpcObserveWithResult("Observe", expectedIdVer19_1_0); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + expected = "Resource [" + fromVersionedIdToObjectId(expectedIdVer19_1_0) + "] conflict with is already registered as SingleObservation [" + fromVersionedIdToObjectId(expectedIdVer19_1) + "]."; + assertEquals(expected, rpcActualResult.get("error").asText()); + // Verify ObserveReadAll + actualValuesReadAll = sendRpcObserveOkWithResultValue("ObserveReadAll", null); + assertTrue(actualValuesReadAll.contains("SingleObservation:/19/1")); + assertTrue(actualValuesReadAll.contains("SingleObservation:/19/0/0")); } /** @@ -240,7 +251,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveResource_ObserveCancelResource_Result_CONTENT_Count_1() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String actualValuesReadAll = sendRpcObserveReadAllWithResult(idVer_3_0_9); assertEquals(1, actualValuesReadAll.split(",").length); @@ -248,7 +259,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO assertTrue(actualValuesReadAll.contains(expected)); // cancel observe "/3_1.2/0/9" - sendRpcObserveWithResultValue("ObserveCancel", idVer_3_0_9); + sendRpcObserveOkWithResultValue("ObserveCancel", idVer_3_0_9); } /** @@ -258,7 +269,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveObject_ObserveCancelOneResource_Result_INTERNAL_SERVER_ERROR_Than_Cancel_ObserveObject_Result_CONTENT_Count_1() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); String actualValuesReadAll = sendRpcObserveReadAllWithResult(objectIdVer_3); assertEquals(1, actualValuesReadAll.split(",").length); @@ -267,34 +278,42 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO // cancel observe "/3_1.2/0/9" ObjectNode rpcActualResult = sendRpcObserveWithResult("ObserveCancel", idVer_3_0_9); - String expectedValue = "for observation path [" + fromVersionedIdToObjectId(objectIdVer_3) + "], that includes this observation path [" + id_3_0_9; + String expectedValue = "Could not find active Observe component with path: " + idVer_3_0_9; assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); assertTrue(rpcActualResult.get("error").asText().contains(expectedValue)); // cancel observe "/3_1.2" - sendRpcObserveWithResultValue("ObserveCancel", objectIdVer_3); + sendRpcObserveOkWithResultValue("ObserveCancel", objectIdVer_3); } /** * Observe {"id":"/3/0/0"} * Observe {"id":"/3/0/9"} - * ObserveCancel {"id":"/3"} - Ok, cnt = 2 + * ObserveCancel {"id":"/3"} - Bad + * ObserveCancel {"/3/0/0"} - Ok + * ObserveCancel {"/3/0/9"} - Ok + * */ @Test public void testObserveResource_ObserveCancelObject_Result_CONTENT_Count_1() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); sendRpcObserveWithWithTwoResource(idVer_3_0_0, idVer_3_0_9); - String rpcActualResul = sendRpcObserveWithResultValue("ObserveReadAll", null); + String rpcActualResul = sendRpcObserveOkWithResultValue("ObserveReadAll", null); assertEquals(2, rpcActualResul.split(",").length); String expected_3_0_0 = "\"SingleObservation:" + fromVersionedIdToObjectId(idVer_3_0_0) + "\""; String expected_3_0_9 = "\"SingleObservation:" + id_3_0_9 + "\""; assertTrue(rpcActualResul.contains(expected_3_0_0)); assertTrue(rpcActualResul.contains(expected_3_0_9)); - // cancel observe "/3_1.2" - String expectedId_3 = objectIdVer_3; - String rpcActualResult = sendRpcObserveWithResultValue("ObserveCancel", expectedId_3); - assertEquals("2", rpcActualResult); + ObjectNode rpcActualResult = sendRpcObserveWithResult("ObserveCancel", objectIdVer_3); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + String expected = "Could not find active Observe component with path: " + objectIdVer_3; + assertEquals(expected, rpcActualResult.get("error").asText()); + // Verify ObserveReadAll + rpcActualResul = sendRpcObserveOkWithResultValue("ObserveReadAll", null); + String expectedReadAll = "[\"SingleObservation:/19/0/0\"]"; + assertTrue(rpcActualResul.contains(expected_3_0_0)); + assertTrue(rpcActualResul.contains(expected_3_0_9)); } /** @@ -305,7 +324,7 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO */ @Test public void testObserveResource_Update_AfterUpdateRegistration() throws Exception { - sendCancelObserveAllWithAwait(deviceId); + sendObserveCancelAllWithAwait(deviceId); int cntUpdate = 3; verify(defaultUplinkMsgHandlerTest, timeout(50000).atLeast(cntUpdate)) @@ -319,37 +338,21 @@ public class RpcLwm2mIntegrationObserveTest extends AbstractRpcLwM2MIntegrationO } private void sendRpcObserveWithWithTwoResource(String expectedId_1, String expectedId_2) throws Exception { - sendRpcObserve("Observe", expectedId_1); - sendRpcObserve("Observe", expectedId_2); + sendRpcObserveOk("Observe", expectedId_1); + sendRpcObserveOk("Observe", expectedId_2); } private String sendRpcObserveReadAllWithResult(String params) throws Exception { - sendRpcObserve("Observe", params); + sendRpcObserveOk("Observe", params); ObjectNode rpcActualResult = sendRpcObserveWithResult("ObserveReadAll", null); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); return rpcActualResult.get("value").asText(); } - private ObjectNode sendRpcObserveWithResult(String method, String params) throws Exception { - String actualResultReadAll = sendRpcObserve(method, params); - return JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - } - - private String sendRpcObserveWithResultValue(String method, String params) throws Exception { - String actualResultReadAll = sendRpcObserve(method, params); - ObjectNode rpcActualResult = JacksonUtil.fromString(actualResultReadAll, ObjectNode.class); - assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); - return rpcActualResult.get("value").asText(); - } - private void sendRpcObserveWithContainsLwM2mSingleResource(String params) throws Exception { - String rpcActualResult = sendRpcObserveWithResultValue("Observe", params); + String rpcActualResult = sendRpcObserveOkWithResultValue("Observe", params); assertTrue(rpcActualResult.contains("LwM2mSingleResource")); assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceId)).get()); } - - private String sendRpcObserve(String method, String params) throws Exception { - return sendObserve(method, params, deviceId); - } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java index 77e427fc77..80221eb5ea 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/downlink/DefaultLwM2mDownlinkMsgHandler.java @@ -68,9 +68,8 @@ import org.eclipse.leshan.core.util.Hex; import org.eclipse.leshan.server.model.LwM2mModelProvider; import org.eclipse.leshan.server.registration.Registration; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.profile.lwm2m.ObjectAttributes; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.server.LwM2mTransportContext; @@ -88,14 +87,11 @@ import org.thingsboard.server.transport.lwm2m.utils.LwM2mValueConverterImpl; import java.util.Arrays; import java.util.Collection; import java.util.Date; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; import java.util.function.Function; @@ -177,24 +173,32 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im } } + /** + * if resource in CompositeObservation is already registered - return BAD REQUEST + */ @Override public void sendObserveRequest(LwM2mClient client, TbLwM2MObserveRequest request, DownlinkRequestCallback callback) { try { validateVersionedId(client, request); LwM2mPath resultIds = new LwM2mPath(request.getObjectId()); - ObserveRequest downlink; - ContentFormat contentFormat = getReadRequestContentFormat(client, request, modelProvider); - if (resultIds.isResourceInstance()) { - downlink = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId(), resultIds.getResourceId(), resultIds.getResourceInstanceId()); - } else if (resultIds.isResource()) { - downlink = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId(), resultIds.getResourceId()); - } else if (resultIds.isObjectInstance()) { - downlink = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId()); + String resourceExisting = checkResourceSingleObservationForExisting(client, resultIds.toString()); + if (StringUtils.isNotBlank(resourceExisting)) { + callback.onValidationError(request.toString(), resourceExisting); } else { - downlink = new ObserveRequest(contentFormat, resultIds.getObjectId()); + ObserveRequest downlink; + ContentFormat contentFormat = getReadRequestContentFormat(client, request, modelProvider); + if (resultIds.isResourceInstance()) { + downlink = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId(), resultIds.getResourceId(), resultIds.getResourceInstanceId()); + } else if (resultIds.isResource()) { + downlink = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId(), resultIds.getResourceId()); + } else if (resultIds.isObjectInstance()) { + downlink = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId()); + } else { + downlink = new ObserveRequest(contentFormat, resultIds.getObjectId()); + } + log.info("[{}] Send observation: {}.", client.getEndpoint(), request.getVersionedId()); + sendSimpleRequest(client, downlink, request.getTimeout(), callback); } - log.info("[{}] Send observation: {}.", client.getEndpoint(), request.getVersionedId()); - sendSimpleRequest(client, downlink, request.getTimeout(), callback); } catch (InvalidRequestException e) { callback.onValidationError(request.toString(), e.getMessage()); } @@ -208,19 +212,7 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im if (observation instanceof SingleObservation) { paths.add("SingleObservation:" + ((SingleObservation) observation).getPath().toString()); } else { - List listPath = ((CompositeObservation) observation).getPaths(); - List pathsComposite = listPath.stream().map(lwM2mPath -> (lwM2mPath.toString())).collect(Collectors.toList()); - Set pathsCompositeSort = new TreeSet<>(); - if (pathsComposite.size() == 1) { - pathsCompositeSort.add("CompositeObservation: [" + pathsComposite.get(0) + "]"); - } else if (pathsComposite.size() > 1) { - List sort = new LinkedList<>(); - sort.addAll(pathsComposite); - sort.set(0, "CompositeObservation: [" + sort.get(0)); - sort.set(pathsComposite.size()-1, sort.get(pathsComposite.size()-1) + "]"); - pathsCompositeSort = new LinkedHashSet<>(sort); - } - paths.addAll(pathsCompositeSort); + paths.add("CompositeObservation: " + ((CompositeObservation) observation).getPaths().toString()); } }); @@ -234,10 +226,15 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im public void sendObserveCompositeRequest(LwM2mClient client, TbLwM2MObserveCompositeRequest request, DownlinkRequestCallback callback) { try { - log.trace("[{}] Send Composite observation: [{}].", client.getEndpoint(), request.getObjectIds()); ContentFormat compositeContentFormat = this.findFirstContentFormatForComposite(client.getClientSupportContentFormats()); - ObserveCompositeRequest downlink = new ObserveCompositeRequest(compositeContentFormat, compositeContentFormat, request.getObjectIds()); - sendCompositeRequest(client, downlink, this.config.getTimeout(), callback); + String resourceExisting = checkResourceForExistingComposite(client, request.getObjectIds()); + if (StringUtils.isNotBlank(resourceExisting)) { + callback.onValidationError(request.toString(), resourceExisting); + } else { + ObserveCompositeRequest downlink = new ObserveCompositeRequest(compositeContentFormat, compositeContentFormat, request.getObjectIds()); + log.trace("[{}] Send ObserveComposite: {}.", client.getEndpoint(), request.getVersionedIds()); + sendCompositeRequest(client, downlink, this.config.getTimeout(), callback); + } } catch (InvalidRequestException e) { callback.onValidationError(request.toString(), e.getMessage()); } @@ -246,44 +243,18 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im @Override public void sendCancelObserveCompositeRequest(LwM2mClient client, TbLwM2MCancelObserveCompositeRequest request, DownlinkRequestCallback callback) { try { - Set observations = context.getServer().getObservationService().getObservations(client.getRegistration()); - List listPath = LwM2mPath.getLwM2mPathList(Arrays.asList(request.getObjectIds())); - Optional observationOpt = Optional.ofNullable(observations.stream().filter(observation -> observation instanceof CompositeObservation && ((CompositeObservation) observation).getPaths().equals(listPath)).findFirst().orElse(null)); - int cnt = 0; - if (observationOpt.isPresent()) { - cnt = context.getServer().getObservationService().cancelCompositeObservations(client.getRegistration(), request.getObjectIds()); + log.trace("[{}] Send CancelObserveComposite: {}.", client.getEndpoint(), request.getVersionedIds()); + int cnt = context.getServer().getObservationService().cancelCompositeObservations(client.getRegistration(), request.getObjectIds()); + if (cnt != 0) { callback.onSuccess(request, cnt); } else { - Set lwPaths = new HashSet<>(); - for (Observation obs : observations) { - LwM2mPath lwPathObs = ((SingleObservation) obs).getPath(); - for (LwM2mPath nodePath : listPath) { - String validNodePath = validatePathObserveCancelAny(nodePath, lwPathObs, client); - if (validNodePath != null) lwPaths.add(validNodePath); - } - }; - for (String nodePath : lwPaths) { - cnt += context.getServer().getObservationService().cancelObservations(client.getRegistration(), nodePath); - } + callback.onValidationError(request.toString(), "Could not find active Observe Composite component with paths: " + Arrays.toString(request.getVersionedIds())); } - callback.onSuccess(request, cnt); - } catch (ThingsboardException e){ + } catch (InvalidRequestException e) { callback.onValidationError(request.toString(), e.getMessage()); } } - private String validatePathObserveCancelAny(LwM2mPath nodePath, LwM2mPath lwPathObs, LwM2mClient client) throws ThingsboardException { - if (nodePath.equals(lwPathObs) || lwPathObs.startWith(nodePath)) { // nodePath = "3", lwPathObs = "3/0/9": cancel for tne all lwPathObs - return lwPathObs.toString(); - } else if (!nodePath.equals(lwPathObs) && nodePath.startWith(lwPathObs)) { - String errorMsg = String.format( - "Unexpected error: There is registration with Endpoint %s for observation path [%s], that includes this observation path [%s]", - client.getRegistration().getEndpoint(), lwPathObs, nodePath); - throw new ThingsboardException(errorMsg, ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } - return null; - } - @Override public void sendDiscoverAllRequest(LwM2mClient client, TbLwM2MDiscoverAllRequest request, DownlinkRequestCallback> callback) { callback.onSuccess(request, Arrays.stream(client.getRegistration().getSortedObjectLinks()).map(Link::toCoreLinkFormat).collect(Collectors.toList())); @@ -334,22 +305,15 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im @Override public void sendCancelObserveRequest(LwM2mClient client, TbLwM2MCancelObserveRequest request, DownlinkRequestCallback callback) { - try{ - validateVersionedId(client, request); - Set observations = context.getServer().getObservationService().getObservations(client.getRegistration()); - int observeCancelCnt = 0; - Set lwPaths = new HashSet<>(); - for (Observation obs : observations) { - LwM2mPath lwPathObs = ((SingleObservation) obs).getPath(); - LwM2mPath nodePath = new LwM2mPath(request.getObjectId()); - String validNodePath = validatePathObserveCancelAny(nodePath, lwPathObs, client); - if (validNodePath != null) lwPaths.add(validNodePath); - }; - for (String nodePath : lwPaths) { - observeCancelCnt += context.getServer().getObservationService().cancelObservations(client.getRegistration(), nodePath); + try { + log.trace("[{}] Send CancelObserve {}.", client.getEndpoint(), request.getVersionedId()); + int cnt = context.getServer().getObservationService().cancelObservations(client.getRegistration(), request.getObjectId()); + if (cnt != 0) { + callback.onSuccess(request, cnt); + } else { + callback.onValidationError(request.toString(), "Could not find active Observe component with path: " + request.getVersionedId()); } - callback.onSuccess(request, observeCancelCnt); - } catch (ThingsboardException e){ + } catch (InvalidRequestException e) { callback.onValidationError(request.toString(), e.getMessage()); } } @@ -416,7 +380,7 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im addAttribute(attributes, OBJECT_VERSION, params.getVer()); // Attachment.OBJECT } - return new LwM2mAttributeSet(attributes); + return new LwM2mAttributeSet(attributes); } @Override @@ -702,13 +666,13 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im addAttribute(attributes, attribute, value, null, null); } - private static void addAttribute(List> attributes, LwM2mAttributeModel attribute, T value, Function converter) { + private static void addAttribute(List> attributes, LwM2mAttributeModel attribute, T value, Function converter) { addAttribute(attributes, attribute, value, null, converter); } - private static void addAttribute(List> attributes, LwM2mAttributeModel attributeName, T value, Predicate filter, Function converter) { + private static void addAttribute(List> attributes, LwM2mAttributeModel attributeName, T value, Predicate filter, Function converter) { if (value != null && ((filter == null) || filter.test(value))) { - T valueConvert = (T) converter != null ? (T) converter.apply(value) : value; + T valueConvert = (T) converter != null ? (T) converter.apply(value) : value; attributes.add(new LwM2mAttribute<>(attributeName, valueConvert)); } } @@ -746,12 +710,12 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im if (resourceModel != null && (pathIds.isResourceInstance() || (pathIds.isResource() && !resourceModel.multiple))) { ContentFormat[] desiredFormats; if (OBJLNK.equals(resourceModel.type)) { - desiredFormats = new ContentFormat[]{ContentFormat.LINK, ContentFormat.CBOR, ContentFormat.SENML_CBOR, ContentFormat.SENML_JSON}; + desiredFormats = new ContentFormat[]{ContentFormat.LINK, ContentFormat.CBOR, ContentFormat.SENML_CBOR, ContentFormat.SENML_JSON}; } else if (OPAQUE.equals(resourceModel.type)) { - desiredFormats = new ContentFormat[]{ContentFormat.OPAQUE, ContentFormat.CBOR, ContentFormat.SENML_CBOR, ContentFormat.SENML_JSON}; - } else { - desiredFormats = new ContentFormat[]{ContentFormat.CBOR, ContentFormat.SENML_CBOR, ContentFormat.SENML_JSON}; - } + desiredFormats = new ContentFormat[]{ContentFormat.OPAQUE, ContentFormat.CBOR, ContentFormat.SENML_CBOR, ContentFormat.SENML_JSON}; + } else { + desiredFormats = new ContentFormat[]{ContentFormat.CBOR, ContentFormat.SENML_CBOR, ContentFormat.SENML_JSON}; + } return findFirstContentFormatForComp(client.getClientSupportContentFormats(), client.getDefaultContentFormat(), desiredFormats); } else { return getContentFormatForComplex(client); @@ -775,6 +739,7 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im throw new RuntimeException("The version " + client.getRegistration().getLwM2mVersion() + " is not supported!"); } } + private String toString(R request) { try { return request != null ? request.toString() : ""; @@ -784,7 +749,7 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im } } - private ContentFormat findFirstContentFormatForComposite (Set clientSupportContentFormats) { + private ContentFormat findFirstContentFormatForComposite(Set clientSupportContentFormats) { ContentFormat contentFormat = findFirstContentFormatForComp(clientSupportContentFormats, null, ContentFormat.SENML_CBOR, ContentFormat.SENML_JSON); if (contentFormat != null) { return contentFormat; @@ -792,8 +757,9 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im throw new RuntimeException("This device does not support Composite Operation"); } } + private static ContentFormat findFirstContentFormatForComp(Set clientSupportContentFormats, ContentFormat defaultValue, ContentFormat... desiredFormats) { - List desiredFormatsList = Arrays.asList(desiredFormats); + List desiredFormatsList = Arrays.asList(desiredFormats); for (ContentFormat c : clientSupportContentFormats) { if (desiredFormatsList.contains(c)) { return c; @@ -801,4 +767,61 @@ public class DefaultLwM2mDownlinkMsgHandler extends LwM2MExecutorAwareService im } return defaultValue; } + + /** + * Check if at least one of the resource objectIds (Composite) in SingleObservation or CompositeObservation is already registered + * @param objectIds + * @return + */ + private String checkResourceForExistingComposite(LwM2mClient client, String[] objectIds) { + List objectIdsList = Arrays.asList(objectIds); + Set observations = context.getServer().getObservationService().getObservations(client.getRegistration()); + for (Observation observation : observations) { + if (observation instanceof SingleObservation singleObs) { + String idSingleOb = singleObs.getPath().toString(); + if (objectIdsList.contains(idSingleOb)) { + return "Resource [" + idSingleOb + "] is already registered as SingleObservation."; + } + } else if (observation instanceof CompositeObservation compObs) { + String paths = compObs.getPaths().toString(); + for (String idCompOb : objectIds) { + if (paths.contains(idCompOb)) { + return "Resource [" + idCompOb + "] is already registered in CompositeObservation."; + } + } + } + } + return null; + } + + /** + * Check if the resource SingleObservation is already registered in CompositeObservation + * Check if the resource SingleObservation is already registered in SingleObservation and (not equals path + * @param objectId + * @return + */ + private String checkResourceSingleObservationForExisting(LwM2mClient client, String objectId) { + Set observations = context.getServer().getObservationService().getObservations(client.getRegistration()); + for (Observation observation : observations) { + if (observation instanceof SingleObservation singleObs) { + LwM2mPath pathSingleOb = singleObs.getPath(); + LwM2mPath pathObjectId = new LwM2mPath(objectId); + if (!pathSingleOb.toString().equals(objectId)) { + List paths = Arrays.asList(pathSingleOb, pathObjectId); + try { + LwM2mPath.validateNotOverlapping(paths); + } catch (IllegalArgumentException e){ + return "Resource [" + objectId + "] conflict with is already registered as SingleObservation [" + pathSingleOb + "]."; + } + } + } + else if (observation instanceof CompositeObservation compObs) { + String paths = compObs.getPaths().toString(); + if (paths.contains(objectId)) { + return "Resource [" + objectId + "] is already registered in CompositeObservation."; + } + } + } + return null; + } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java index 8fae6a478e..06b0d62bde 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbInMemoryRegistrationStore.java @@ -301,6 +301,12 @@ public class TbInMemoryRegistrationStore implements RegistrationStore, Startable try { lock.writeLock().lock(); Observation observation = unsafeGetObservation(observationId); + if (observation instanceof SingleObservation){ + log.trace("(SingleObservation) removeObservation: [{}]", ((SingleObservation)observation).getPath()); + } else { + log.trace("(CompositeObservation) removeObservation: [{}]", ((CompositeObservation)observation).getPaths()); + } + if (observation != null && registrationId.equals(observation.getRegistrationId())) { unsafeRemoveObservation(observationId); return observation; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java index 61d751d64d..df1a86d2eb 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2mUplinkMsgHandler.java @@ -630,17 +630,17 @@ public class DefaultLwM2mUplinkMsgHandler extends LwM2MExecutorAwareService impl * @param registration - Registration LwM2M Client */ public void updateAttrTelemetry(Registration registration, String path) { + log.trace("UpdateAttrTelemetry paths [{}]", path); try { ResultsAddKeyValueProto results = this.getParametersFromProfile(registration, path); - if (path.equals("/3_1.2/0/9")) { - log.info("UpdateTelemetry paths [{}] key: [{}] value [{}]", path, results.getResultTelemetries().get(0).getKey(), results.getResultTelemetries().get(0).getLongV()); - } SessionInfoProto sessionInfo = this.getSessionInfoOrCloseSession(registration); if (results != null && sessionInfo != null) { if (results.getResultAttributes().size() > 0) { + log.trace("UpdateAttribute paths [{}] value [{}]", path, results.getResultAttributes().get(0).toString()); this.helper.sendParametersOnThingsboardAttribute(results.getResultAttributes(), sessionInfo); } if (results.getResultTelemetries().size() > 0) { + log.trace("UpdateTelemetry paths [{}] value [{}]", path, results.getResultTelemetries().get(0).toString()); this.helper.sendParametersOnThingsboardTelemetry(results.getResultTelemetries(), sessionInfo); } } From 0c4e06a0f948c46eaa4d2014a09ad2accc5d2283 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 16 Sep 2024 10:31:12 +0200 Subject: [PATCH 24/32] minor refactoring due to comments --- .../actors/ruleChain/RuleNodeActorMessageProcessor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java index 46ffb6e894..75499b9a8d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleNodeActorMessageProcessor.java @@ -58,7 +58,7 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor Date: Mon, 16 Sep 2024 11:53:12 +0200 Subject: [PATCH 25/32] fixed NPE in subscription service --- .../subscription/DefaultTbLocalSubscriptionService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index c713bece6b..4043b519da 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -334,7 +334,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } private void onTimeSeriesUpdate(UUID entityId, List data, TbCallback callback) { - entityUpdates.get(entityId).timeSeriesUpdateTs = System.currentTimeMillis(); + getEntityUpdatesInfo(entityId).timeSeriesUpdateTs = System.currentTimeMillis(); processSubscriptionData(entityId, sub -> TbSubscriptionType.TIMESERIES.equals(sub.getType()), s -> { @@ -371,7 +371,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer } private void onAttributesUpdate(UUID entityId, String scope, List data, TbCallback callback) { - entityUpdates.get(entityId).attributesUpdateTs = System.currentTimeMillis(); + getEntityUpdatesInfo(entityId).attributesUpdateTs = System.currentTimeMillis(); processSubscriptionData(entityId, sub -> TbSubscriptionType.ATTRIBUTES.equals(sub.getType()), s -> { @@ -639,4 +639,8 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer throw new TbRateLimitsException(message); } + private TbEntityUpdatesInfo getEntityUpdatesInfo(UUID entityId) { + return entityUpdates.computeIfAbsent(entityId, id -> new TbEntityUpdatesInfo(0)); + } + } From af0ed296443a5341f732fd6589fef17f8d8cae26 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 16 Sep 2024 14:30:00 +0300 Subject: [PATCH 26/32] fixed upgrade script --- .../server/service/install/SqlDatabaseUpgradeService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index bfe389d6ff..32833a63ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -126,7 +126,8 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService try { connection.createStatement().execute("UPDATE rule_node SET " + "configuration = CASE " + - " WHEN (configuration::jsonb ->> 'persistAlarmRulesState')::boolean = false THEN jsonb_set(configuration::jsonb, '{fetchAlarmRulesStateOnStart}', 'false'::jsonb)::varchar " + + " WHEN (configuration::jsonb ->> 'persistAlarmRulesState') = 'false'" + + " THEN (configuration::jsonb || '{\"fetchAlarmRulesStateOnStart\": \"false\"}'::jsonb)::varchar " + " ELSE configuration " + "END, " + "configuration_version = 1 " + From 7e70e79f2fe3424c1ac4a63161c842973fdd8a30 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 16 Sep 2024 15:39:06 +0300 Subject: [PATCH 27/32] added log --- .../server/service/install/SqlDatabaseUpgradeService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 32833a63ba..b3d20ff5e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -134,6 +134,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService "WHERE type = 'org.thingsboard.rule.engine.profile.TbDeviceProfileNode' " + "AND configuration_version < 1;"); } catch (Exception e) { + log.warn("Failed to execute update script for device profile rule nodes due to: ", e); } }); break; From 8aa67eeb443236513e785d786e5412d232c623e1 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 16 Sep 2024 18:04:32 +0300 Subject: [PATCH 28/32] Fix alarms unassigning when user is deleted --- .../entitiy/alarm/DefaultTbAlarmService.java | 21 ++++++----- .../service/entitiy/alarm/TbAlarmService.java | 5 +-- .../AlarmsUnassignTaskProcessor.java | 7 ++-- .../housekeeper/HousekeeperServiceTest.java | 35 ++++++++++++------- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index e54f93be8f..8a7e285c8e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -39,10 +39,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import java.util.ArrayList; import java.util.List; @Service @@ -176,20 +174,20 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) { - List totalAlarmIds = new ArrayList<>(); - PageLink pageLink = new PageLink(100, 0, null, new SortOrder("id", SortOrder.Direction.ASC)); + public int unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) { + int count = 0; + PageLink pageLink = new PageLink(100); while (true) { PageData pageData = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, pageLink); - List alarmIds = pageData.getData(); - if (alarmIds.isEmpty()) { + List alarms = pageData.getData(); + processAlarmsUnassignment(tenantId, userId, userTitle, alarms, unassignTs); + + count += alarms.size(); + if (!pageData.hasNext()) { break; } - processAlarmsUnassignment(tenantId, userId, userTitle, alarmIds, unassignTs); - totalAlarmIds.addAll(alarmIds); - pageLink = pageLink.nextPageLink(); } - return totalAlarmIds; + return count; } @Override @@ -245,4 +243,5 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb log.error("Failed to save alarm comment", e); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index c975291b58..5a55c9990f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -19,12 +19,9 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; -import java.util.List; - public interface TbAlarmService { Alarm save(Alarm entity, User user) throws ThingsboardException; @@ -41,7 +38,7 @@ public interface TbAlarmService { AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws ThingsboardException; - List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs); + int unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs); Boolean delete(Alarm alarm, User user); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index ba596da500..326cce6c68 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -20,12 +20,9 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.housekeeper.AlarmsUnassignHousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; -import java.util.List; - @Component @RequiredArgsConstructor @Slf4j @@ -35,8 +32,8 @@ public class AlarmsUnassignTaskProcessor extends HousekeeperTaskProcessor alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); - log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); + int count = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), count); } @Override diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 06740e2851..5420998e2e 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -55,6 +55,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbNodeConnectionType; +import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -184,23 +185,31 @@ public class HousekeeperServiceTest extends AbstractControllerTest { Device device = createDevice("test", "test"); UserId userId = customerUserId; createRelatedData(userId); - Alarm alarm = Alarm.builder() - .type("test") - .tenantId(tenantId) - .originator(device.getId()) - .severity(AlarmSeverity.MAJOR) - .build(); - alarm = doPost("/api/alarm", alarm, Alarm.class); - AlarmId alarmId = alarm.getId(); - alarm = doPost("/api/alarm/" + alarmId + "/assign/" + userId, "", Alarm.class); - assertThat(alarm.getAssigneeId()).isEqualTo(userId); - assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(100)).getData()).isNotEmpty(); + + List alarms = new ArrayList<>(); + int count = 112; + for (int i = 0; i < count; i++) { + Alarm alarm = Alarm.builder() + .type("test" + i) + .tenantId(tenantId) + .originator(device.getId()) + .severity(AlarmSeverity.MAJOR) + .build(); + alarm = doPost("/api/alarm", alarm, Alarm.class); + AlarmId alarmId = alarm.getId(); + alarm = doPost("/api/alarm/" + alarmId + "/assign/" + userId, "", Alarm.class); + assertThat(alarm.getAssigneeId()).isEqualTo(userId); + alarms.add(alarmId); + } + PageData assignedAlarms = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(Integer.MAX_VALUE)); + assertThat(assignedAlarms.getTotalElements()).isEqualTo(count); + assertThat(assignedAlarms.getData()).containsAll(alarms); doDelete("/api/user/" + userId).andExpect(status().isOk()); - await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { verifyNoRelatedData(userId); - assertThat(alarmService.findAlarmById(tenantId, alarmId).getAssigneeId()).isNull(); + assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(1)).getTotalElements()).isZero(); }); } From dd3936ca666dc9b9ba867b002e553bb5d971c16e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 17 Sep 2024 11:34:19 +0300 Subject: [PATCH 29/32] Performance improvements for processing alarms unassigning task --- .../entitiy/alarm/DefaultTbAlarmService.java | 43 ++++++------------- .../service/entitiy/alarm/TbAlarmService.java | 5 ++- .../AlarmsUnassignTaskProcessor.java | 33 ++++++++++++-- .../alarm/DefaultTbAlarmServiceTest.java | 7 +-- .../housekeeper/HousekeeperServiceTest.java | 12 +++--- .../server/dao/alarm/AlarmService.java | 3 +- .../AlarmsUnassignHousekeeperTask.java | 20 ++++++++- .../server/dao/alarm/AlarmDao.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 7 +-- .../server/dao/sql/alarm/AlarmRepository.java | 19 +++++++- .../server/dao/sql/alarm/JpaAlarmDao.java | 12 ++++-- 11 files changed, 105 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 8a7e285c8e..d9469bd2db 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -37,11 +37,10 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import java.util.List; +import java.util.UUID; @Service @AllArgsConstructor @@ -174,20 +173,20 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public int unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) { - int count = 0; - PageLink pageLink = new PageLink(100); - while (true) { - PageData pageData = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, pageLink); - List alarms = pageData.getData(); - processAlarmsUnassignment(tenantId, userId, userTitle, alarms, unassignTs); - - count += alarms.size(); - if (!pageData.hasNext()) { - break; + public void unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, List alarms, long unassignTs) { + for (UUID alarmId : alarms) { + log.trace("[{}] Unassigning alarm {} from user {}", tenantId, alarmId, userId); + AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(tenantId, new AlarmId(alarmId), unassignTs); + if (!result.isSuccessful()) { + log.error("[{}] Cannot unassign alarm {} from user {}", tenantId, alarmId, userId); + continue; + } + if (result.isModified()) { + String comment = String.format("Alarm was unassigned because user %s - was deleted", userTitle); + addSystemAlarmComment(result.getAlarm(), null, "ASSIGN", comment); + logEntityActionService.logEntityAction(result.getAlarm().getTenantId(), result.getAlarm().getOriginator(), result.getAlarm(), result.getAlarm().getCustomerId(), ActionType.ALARM_UNASSIGNED, null); } } - return count; } @Override @@ -202,22 +201,6 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb return ts > 0 ? ts : System.currentTimeMillis(); } - private void processAlarmsUnassignment(TenantId tenantId, UserId userId, String userTitle, List alarmIds, long unassignTs) { - for (AlarmId alarmId : alarmIds) { - log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, userId); - AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(tenantId, alarmId, unassignTs); - if (!result.isSuccessful()) { - log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, userId); - continue; - } - if (result.isModified()) { - String comment = String.format("Alarm was unassigned because user %s - was deleted", userTitle); - addSystemAlarmComment(result.getAlarm(), null, "ASSIGN", comment); - logEntityActionService.logEntityAction(result.getAlarm().getTenantId(), result.getAlarm().getOriginator(), result.getAlarm(), result.getAlarm().getCustomerId(), ActionType.ALARM_UNASSIGNED, null); - } - } - } - private void addSystemAlarmComment(Alarm alarm, User user, String subType, String commentText) { addSystemAlarmComment(alarm, user, subType, commentText, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index 5a55c9990f..7ba9bfc87c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -22,6 +22,9 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import java.util.List; +import java.util.UUID; + public interface TbAlarmService { Alarm save(Alarm entity, User user) throws ThingsboardException; @@ -38,7 +41,7 @@ public interface TbAlarmService { AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws ThingsboardException; - int unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs); + void unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, List alarms, long unassignTs); Boolean delete(Alarm alarm, User user); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 326cce6c68..9d6eb2860d 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -20,20 +20,47 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.housekeeper.AlarmsUnassignHousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; +import java.util.List; +import java.util.UUID; + @Component @RequiredArgsConstructor @Slf4j public class AlarmsUnassignTaskProcessor extends HousekeeperTaskProcessor { - private final TbAlarmService alarmService; + private final TbAlarmService tbAlarmService; + private final AlarmService alarmService; @Override public void process(AlarmsUnassignHousekeeperTask task) throws Exception { - int count = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); - log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), count); + TenantId tenantId = task.getTenantId(); + UserId userId = (UserId) task.getEntityId(); + if (task.getAlarms() == null) { + AlarmId lastId = null; + long lastCreatedTime = 0; + while (true) { + List> alarms = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, lastCreatedTime, lastId, 64); + if (alarms.isEmpty()) { + break; + } + housekeeperClient.submitTask(new AlarmsUnassignHousekeeperTask(tenantId, userId, task.getUserTitle(), alarms.stream().map(TbPair::getFirst).toList())); + + TbPair last = alarms.get(alarms.size() - 1); + lastId = new AlarmId(last.getFirst()); + lastCreatedTime = last.getSecond(); + log.debug("[{}][{}] Submitted task for unassigning {} alarms", tenantId, userId, alarms.size()); + } + } else { + tbAlarmService.unassignDeletedUserAlarms(tenantId, userId, task.getUserTitle(), task.getAlarms(), task.getTs()); + log.debug("[{}][{}] Unassigned {} alarms", tenantId, userId, task.getAlarms().size()); + } } @Override diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 902eef918f..60a0ac3bf6 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.edge.EdgeService; @@ -46,7 +45,6 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; -import java.util.Collections; import java.util.List; import java.util.UUID; @@ -164,16 +162,13 @@ public class DefaultTbAlarmServiceTest { AlarmInfo alarm = new AlarmInfo(); alarm.setId(new AlarmId(UUID.randomUUID())); - when(alarmService.findAlarmIdsByAssigneeId(any(), any(), any())) - .thenReturn(new PageData<>(List.of(alarm.getId()), 0, 1, false)) - .thenReturn(new PageData<>(Collections.EMPTY_LIST, 0, 0, false)); when(alarmSubscriptionService.unassignAlarm(any(), any(), anyLong())) .thenReturn(AlarmApiCallResult.builder().successful(true).modified(true).alarm(alarm).build()); User user = new User(); user.setEmail("testEmail@gmail.com"); user.setId(new UserId(UUID.randomUUID())); - service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), System.currentTimeMillis()); + service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), List.of(alarm.getUuidId()), System.currentTimeMillis()); ObjectNode commentNode = JacksonUtil.newObjectNode(); commentNode.put("subtype", "ASSIGN"); diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 5420998e2e..d3b1916def 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -55,8 +55,6 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbNodeConnectionType; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -64,6 +62,7 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.alarm.AlarmService; @@ -201,15 +200,16 @@ public class HousekeeperServiceTest extends AbstractControllerTest { assertThat(alarm.getAssigneeId()).isEqualTo(userId); alarms.add(alarmId); } - PageData assignedAlarms = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(Integer.MAX_VALUE)); - assertThat(assignedAlarms.getTotalElements()).isEqualTo(count); - assertThat(assignedAlarms.getData()).containsAll(alarms); + List assignedAlarms = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, 0, null, 5000).stream() + .map(TbPair::getFirst).map(AlarmId::new).toList(); + assertThat(assignedAlarms).size().isEqualTo(count); + assertThat(assignedAlarms).containsAll(alarms); doDelete("/api/user/" + userId).andExpect(status().isOk()); await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { verifyNoRelatedData(userId); - assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(1)).getTotalElements()).isZero(); + assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, 0, null, 5000)).size().isZero(); }); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 5bf6b95853..256d10465d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -107,7 +107,7 @@ public interface AlarmService extends EntityDaoService { PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds); - PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); + List> findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, long createdTimeOffset, AlarmId idOffset, int limit); List> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit); @@ -118,4 +118,5 @@ public interface AlarmService extends EntityDaoService { long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); PageData findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java index 5911418dc3..b84e8cd742 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -21,6 +21,11 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.ToString; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; + +import java.util.List; +import java.util.UUID; @Data @ToString(callSuper = true) @@ -29,10 +34,21 @@ import org.thingsboard.server.common.data.User; public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { private String userTitle; + private List alarms; protected AlarmsUnassignHousekeeperTask(User user) { - super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); - this.userTitle = user.getTitle(); + this(user.getTenantId(), user.getId(), user.getTitle(), null); + } + + public AlarmsUnassignHousekeeperTask(TenantId tenantId, UserId userId, String userTitle, List alarms) { + super(tenantId, userId, HousekeeperTaskType.UNASSIGN_ALARMS); + this.userTitle = userTitle; + this.alarms = alarms; + } + + @Override + public String getDescription() { + return super.getDescription() + (alarms != null ? " (" + alarms + ")" : ""); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index 570256e98c..07b670f835 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -79,7 +79,7 @@ public interface AlarmDao extends Dao { PageData findAlarmsIdsByEndTsBeforeAndTenantId(Long time, TenantId tenantId, PageLink pageLink); - PageData findAlarmIdsByAssigneeId(TenantId tenantId, UUID userId, PageLink pageLink); + PageData> findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, long createdTimeOffset, AlarmId idOffset, int limit); PageData> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index f0fc18c827..7e9cabf5b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -308,10 +308,10 @@ public class BaseAlarmService extends AbstractCachedEntityService findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink) { - log.trace("[{}] Executing findAlarmIdsByAssigneeId [{}]", tenantId, userId); + public List> findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, long createdTimeOffset, AlarmId idOffset, int limit) { + log.trace("[{}] Executing findAlarmIdsByAssigneeId [{}][{}]", tenantId, userId, idOffset); validateId(userId, id -> "Incorrect userId " + id); - return alarmDao.findAlarmIdsByAssigneeId(tenantId, userId.getId(), pageLink); + return alarmDao.findAlarmIdsByAssigneeId(tenantId, userId, createdTimeOffset, idOffset, limit).getData(); } @Override @@ -476,4 +476,5 @@ public class BaseAlarmService extends AbstractCachedEntityService { @Query(value = "SELECT a FROM AlarmInfoEntity a WHERE a.tenantId = :tenantId AND a.id = :alarmId") AlarmInfoEntity findAlarmInfoById(@Param("tenantId") UUID tenantId, @Param("alarmId") UUID alarmId); - @Query("SELECT a.id FROM AlarmEntity a WHERE a.tenantId = :tenantId AND a.assigneeId = :assigneeId") - Page findAlarmIdsByAssigneeId(@Param("tenantId") UUID tenantId, @Param("assigneeId") UUID assigneeId, Pageable pageable); + // using Slice so that count query is not executed + @Query("SELECT new org.thingsboard.server.common.data.util.TbPair(a.id, a.createdTime) " + + "FROM AlarmEntity a WHERE a.tenantId = :tenantId AND a.assigneeId = :assigneeId") + Slice> findAlarmIdsByAssigneeId(@Param("tenantId") UUID tenantId, + @Param("assigneeId") UUID assigneeId, + Pageable pageable); + + // using Slice so that count query is not executed + @Query("SELECT new org.thingsboard.server.common.data.util.TbPair(a.id, a.createdTime) " + + "FROM AlarmEntity a WHERE a.tenantId = :tenantId AND a.assigneeId = :assigneeId " + + "AND (a.createdTime > :createdTimeOffset OR " + + "(a.createdTime = :createdTimeOffset AND a.id > :idOffset))") + Slice> findAlarmIdsByAssigneeId(@Param("tenantId") UUID tenantId, + @Param("assigneeId") UUID assigneeId, + @Param("createdTimeOffset") long createdTimeOffset, + @Param("idOffset") UUID idOffset, + Pageable pageable); // using Slice so that count query is not executed @Query("SELECT new org.thingsboard.server.common.data.util.TbPair(a.id, a.createdTime) " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 133eb70521..b4c1bf7913 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -296,9 +296,15 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public PageData findAlarmIdsByAssigneeId(TenantId tenantId, UUID userId, PageLink pageLink) { - return DaoUtil.pageToPageData(alarmRepository.findAlarmIdsByAssigneeId(tenantId.getId(), userId, DaoUtil.toPageable(pageLink))) - .mapData(AlarmId::new); + public PageData> findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, long createdTimeOffset, AlarmId idOffset, int limit) { + Slice> result; + Pageable pageRequest = toPageable(new PageLink(limit), List.of(SortOrder.of("createdTime", ASC), SortOrder.of("id", ASC))); + if (idOffset == null) { + result = alarmRepository.findAlarmIdsByAssigneeId(tenantId.getId(), userId.getId(), pageRequest); + } else { + result = alarmRepository.findAlarmIdsByAssigneeId(tenantId.getId(), userId.getId(), createdTimeOffset, idOffset.getId(), pageRequest); + } + return DaoUtil.pageToPageData(result); } @Override From f10f63bb8a281cf883ae4881395b3f241c47a0fa Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 17 Sep 2024 11:25:50 +0200 Subject: [PATCH 30/32] fixed concurrent modification in TopicService --- .../thingsboard/server/queue/discovery/TopicService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 960be231c6..b73c3c1c72 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import java.util.HashMap; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; @Service public class TopicService { @@ -30,8 +30,8 @@ public class TopicService { @Value("${queue.prefix:}") private String prefix; - private Map tbCoreNotificationTopics = new HashMap<>(); - private Map tbRuleEngineNotificationTopics = new HashMap<>(); + private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); /** * Each Service should start a consumer for messages that target individual service instance based on serviceId. From cc62caf1fd01dd92b801dd5a9b297d581257e718 Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 17 Sep 2024 15:24:12 +0300 Subject: [PATCH 31/32] lwm2m: fix bug Observe/Composite - MSA tests3 --- .../lwm2m/AbstractLwm2mClientTest.java | 174 ++++++++++++------ .../lwm2m/client/LwM2MTestClient.java | 8 + .../lwm2m/client/Lwm2mTestHelper.java | 39 ++-- .../lwm2m/rpc/Lwm2mObserveCompositeTest.java | 2 +- .../lwm2m/rpc/Lwm2mObserveTest.java | 2 +- 5 files changed, 158 insertions(+), 67 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/AbstractLwm2mClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/AbstractLwm2mClientTest.java index b60eb15b5c..bb5c642e91 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/AbstractLwm2mClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/AbstractLwm2mClientTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.msa.connectivity.lwm2m; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Sets; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonParser; @@ -25,6 +26,7 @@ import org.eclipse.leshan.client.object.Security; import org.eclipse.leshan.core.ResponseCode; import org.eclipse.leshan.core.util.Hex; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.Assert; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Device; @@ -48,17 +50,21 @@ import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTrans import org.thingsboard.server.common.data.device.profile.lwm2m.OtherConfiguration; import org.thingsboard.server.common.data.device.profile.lwm2m.TelemetryMappingConfiguration; import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBootstrapServerCredential; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.WsClient; import org.thingsboard.server.msa.connectivity.lwm2m.client.LwM2MTestClient; import org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState; +import org.thingsboard.server.msa.mapper.WsTelemetryResponse; import java.net.ServerSocket; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -73,7 +79,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.eclipse.leshan.client.object.Security.psk; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.testng.AssertJUnit.assertEquals; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.CLIENT_ENDPOINT_PSK; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.CLIENT_LWM2M_SETTINGS; @@ -84,6 +89,11 @@ import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelp import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_REGISTRATION_SUCCESS; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_UPDATE_SUCCESS; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.OBSERVE_ATTRIBUTES_WITH_PARAMS; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.RESOURCE_ID_NAME_19_0_0; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.RESOURCE_ID_NAME_19_0_2; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.RESOURCE_ID_NAME_19_1_0; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.RESOURCE_ID_NAME_3_14; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.RESOURCE_ID_NAME_3_9; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.SECURE_URI; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.SECURITY_NO_SEC; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.resources; @@ -101,16 +111,16 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { public final Set expectedStatusesRegistrationLwm2mSuccess = new HashSet<>(Arrays.asList(ON_INIT, ON_REGISTRATION_STARTED, ON_REGISTRATION_SUCCESS)); public void createLwm2mDevicesForConnectNoSec(String name, Lwm2mDevicesForTest devicesForTest) throws Exception { - String clientEndpoint = name + "-" + RandomStringUtils.randomAlphanumeric(7); + String clientEndpoint = name + "-" + RandomStringUtils.randomAlphanumeric(7); LwM2MDeviceCredentials deviceCredentials = getDeviceCredentialsNoSec(createNoSecClientCredentials(clientEndpoint)); - Device lwM2MDeviceTest = createDeviceWithCredentials(deviceCredentials, clientEndpoint, devicesForTest.getLwm2mDeviceProfile().getId()); + Device lwM2MDeviceTest = createDeviceWithCredentials(deviceCredentials, clientEndpoint, devicesForTest.getLwm2mDeviceProfile().getId()); LwM2MTestClient lwM2MTestClient = createNewClient(SECURITY_NO_SEC, clientEndpoint, executor); devicesForTest.setLwM2MDeviceTest(lwM2MDeviceTest); devicesForTest.setLwM2MTestClient(lwM2MTestClient); } public void createLwm2mDevicesForConnectPsk(Lwm2mDevicesForTest devicesForTest) throws Exception { - String clientEndpoint = CLIENT_ENDPOINT_PSK +"-" + RandomStringUtils.randomAlphanumeric(7); + String clientEndpoint = CLIENT_ENDPOINT_PSK + "-" + RandomStringUtils.randomAlphanumeric(7); String identity = CLIENT_PSK_IDENTITY; String keyPsk = CLIENT_PSK_KEY; PSKClientCredential clientCredentials = new PSKClientCredential(); @@ -128,54 +138,93 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { devicesForTest.setLwM2MTestClient(lwM2MTestClient); } - public void observeResource_Update_AfterUpdateRegistration_test(LwM2MTestClient lwM2MTestClient, String deviceIdStr) throws Exception { + /** + * Observe {"id":"/3/0/0"} + * Observe {"id":"/3/0/9"} + * ObserveCancel {"id":"/3"} - Bad + * ObserveCancel {"/3/0/0"} - Ok + * ObserveCancelAl - Ok + * + * @param lwM2MTestClient + * @param deviceId + * @throws Exception + */ + public void observeResource_Update_AfterUpdateRegistration_test(LwM2MTestClient lwM2MTestClient, DeviceId deviceId) throws Exception { awaitUpdateRegistrationSuccess(lwM2MTestClient, 5); - sendCancelObserveAllWithAwait(deviceIdStr); + sendCancelObserveAllWithAwait(deviceId.toString()); awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); + long tsBefore = Instant.now().toEpochMilli(); String param = "/3_1.2/0/9"; - sendRpcObserveWithContainsLwM2mSingleResource(param, deviceIdStr); - awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); - sendCancelObserveAllWithAwait(deviceIdStr); + sendRpcObserveWithContainsLwM2mSingleResource(param, deviceId.toString(), 1); awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); - sendRpcObserveWithContainsLwM2mSingleResource(param, deviceIdStr); + checkLatestTelemetryUploaded(deviceId, tsBefore, RESOURCE_ID_NAME_3_9); + param = "/3_1.2/0/0"; + sendRpcObserveWithContainsLwM2mSingleResource(param, deviceId.toString(), 2); awaitUpdateRegistrationSuccess(lwM2MTestClient, 2); + param = "/3_1.2"; + String expected = "Could not find active Observe component with path: " + param; + String actual = sendObserveCancel_BadRequest("ObserveCancel", param, deviceId.toString()); + assertEquals(expected, actual); + param = "/3_1.2/0/0"; + sendObserveCancel_Ok("ObserveCancel", param, deviceId.toString()); + sendCancelObserveAllWithAwait(deviceId.toString()); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); } - public void observeCompositeResource_Update_AfterUpdateRegistration_test(LwM2MTestClient lwM2MTestClient, String deviceIdStr) throws Exception { + + public void observeCompositeResource_Update_AfterUpdateRegistration_test(LwM2MTestClient lwM2MTestClient, DeviceId deviceId) throws Exception { + String id_3_0_9 = "/3/0/9"; + String id_3_0_14 = "/3/0/14"; + String id_19_0_0 = "/19/0/0"; + String id_19_1_0 = "/19/1/0"; + String id_19_0_2 = "/19/0/2"; + awaitUpdateRegistrationSuccess(lwM2MTestClient, 5); - sendCancelObserveAllWithAwait(deviceIdStr); + sendCancelObserveAllWithAwait(deviceId.toString()); awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); - String expectedKey3_0_9 = "batteryLevel"; -// String expectedKey3_0_14 = "UtfOffset"; -// String expectedKey19_0_0 = "dataRead"; -// String expectedKey19_1_0 = "dataWrite"; -// String expectedKeys = "[\"" + expectedKey3_0_9 + "\", \"" + expectedKey3_0_14 + "\", \"" + expectedKey19_0_0 + "\", \"" + expectedKey19_1_0 + "\", \"" + expectedKey3_0_9 + "\"]"; - String expectedKeys = "[\"" + expectedKey3_0_9 + "\"]"; - sendRpcObserveCompositeWithContainsLwM2mSingleResource(expectedKeys, deviceIdStr); + long tsBefore = Instant.now().toEpochMilli(); + String expectedKeys = "[\"" + RESOURCE_ID_NAME_3_9 + "\", \"" + RESOURCE_ID_NAME_3_14 + "\", \"" + RESOURCE_ID_NAME_19_0_0 + "\", \"" + RESOURCE_ID_NAME_19_0_2 + "\", \"" +RESOURCE_ID_NAME_19_1_0 + "\"]"; + String actualResult = sendRpcObserveCompositeWithResultValue(expectedKeys, deviceId.toString()); + assertTrue(actualResult.contains(id_3_0_9 + "=LwM2mSingleResource")); + assertTrue(actualResult.contains(id_3_0_14 + "=LwM2mSingleResource")); + assertTrue(actualResult.contains(id_19_0_0 + "=LwM2mMultipleResource")); + assertTrue(actualResult.contains(id_19_1_0 + "=LwM2mMultipleResource")); + assertTrue(actualResult.contains(id_19_0_2 + "=LwM2mSingleResource")); + // ObserveComposite: - verify + ObjectNode rpcActualResultBefore = sendRpcObserve("ObserveReadAll", null, deviceId.toString()); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultBefore.get("result").asText()); + JsonElement element = JsonParser.parseString(rpcActualResultBefore.get("value").asText()); + assertEquals(1, ((JsonArray) element).size()); + actualResult = ((JsonArray) element).asList().get(0).getAsString(); + assertTrue(actualResult.contains("CompositeObservation:")); + checkLatestTelemetryUploaded(deviceId, tsBefore, RESOURCE_ID_NAME_3_9, RESOURCE_ID_NAME_3_14, + RESOURCE_ID_NAME_19_0_0, RESOURCE_ID_NAME_19_0_2, RESOURCE_ID_NAME_19_1_0); + awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); - sendCancelObserveAllWithAwait(deviceIdStr); + sendCancelObserveAllWithAwait(deviceId.toString()); awaitUpdateRegistrationSuccess(lwM2MTestClient, 1); - sendRpcObserveCompositeWithContainsLwM2mSingleResource(expectedKeys, deviceIdStr); - awaitUpdateRegistrationSuccess(lwM2MTestClient, 2); + assertEquals(0, (Object) Optional.ofNullable(getCntObserveAll(deviceId.toString())).get()); } + public void basicTestConnection(LwM2MTestClient lwM2MTestClient, String alias) throws Exception { - LwM2MClientState finishState = ON_REGISTRATION_SUCCESS; - await(alias + " - " + ON_REGISTRATION_STARTED) - .atMost(40, TimeUnit.SECONDS) - .until(() -> { - log.warn("msa basicTestConnection started -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); - return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_REGISTRATION_STARTED); - }); - await(alias + " - " + ON_UPDATE_SUCCESS) - .atMost(40, TimeUnit.SECONDS) - .until(() -> { - log.warn("msa basicTestConnection update -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); - return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_UPDATE_SUCCESS); - }); - assertThat(lwM2MTestClient.getClientStates()).containsAll(expectedStatusesRegistrationLwm2mSuccess); + LwM2MClientState finishState = ON_REGISTRATION_SUCCESS; + await(alias + " - " + ON_REGISTRATION_STARTED) + .atMost(40, TimeUnit.SECONDS) + .until(() -> { + log.warn("msa basicTestConnection started -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); + return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_REGISTRATION_STARTED); + }); + await(alias + " - " + ON_UPDATE_SUCCESS) + .atMost(40, TimeUnit.SECONDS) + .until(() -> { + log.warn("msa basicTestConnection update -> finishState: [{}] states: {}", finishState, lwM2MTestClient.getClientStates()); + return lwM2MTestClient.getClientStates().contains(finishState) || lwM2MTestClient.getClientStates().contains(ON_UPDATE_SUCCESS); + }); + assertThat(lwM2MTestClient.getClientStates()).containsAll(expectedStatusesRegistrationLwm2mSuccess); } + public LwM2MTestClient createNewClient(Security security, String endpoint, ScheduledExecutorService executor) throws Exception { this.executor = executor; @@ -187,7 +236,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { return lwM2MTestClient; } - protected void destroyAfter(Lwm2mDevicesForTest devicesForTest){ + protected void destroyAfter(Lwm2mDevicesForTest devicesForTest) { clientDestroy(devicesForTest.getLwM2MTestClient()); deviceDestroy(devicesForTest.getLwM2MDeviceTest()); deviceProfileDestroy(devicesForTest.getLwm2mDeviceProfile()); @@ -205,6 +254,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { log.error("Failed client Destroy", e); } } + protected void deviceDestroy(Device lwM2MDeviceTest) { try { if (lwM2MDeviceTest != null) { @@ -269,7 +319,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { return deviceProfile; } - protected void deviceProfileDestroy(DeviceProfile lwm2mDeviceProfile){ + protected void deviceProfileDestroy(DeviceProfile lwm2mDeviceProfile) { try { if (lwm2mDeviceProfile != null) { testRestClient.deleteDeviceProfileIfExists(lwm2mDeviceProfile); @@ -280,7 +330,7 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { } protected Device createDeviceWithCredentials(LwM2MDeviceCredentials deviceCredentials, String clientEndpoint, DeviceProfileId profileId) throws Exception { - Device device = createDevice(deviceCredentials, clientEndpoint, profileId); + Device device = createDevice(deviceCredentials, clientEndpoint, profileId); return device; } @@ -352,25 +402,39 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { return bootstrapCredentials; } + protected String sendObserveCancel_BadRequest(String method, String params, String deviceIdStr) throws Exception { + ObjectNode rpcActualResult = sendRpcObserve(method, params, deviceIdStr); + assertEquals(ResponseCode.BAD_REQUEST.getName(), rpcActualResult.get("result").asText()); + return rpcActualResult.get("error").asText(); + } + + protected void sendObserveCancel_Ok(String method, String params, String deviceIdStr) throws Exception { + ObjectNode rpcActualResult = sendRpcObserve(method, params, deviceIdStr); + assertEquals(ResponseCode.CONTENT.getName(), rpcActualResult.get("result").asText()); + assertEquals("1", rpcActualResult.get("value").asText()); + } + protected void sendCancelObserveAllWithAwait(String deviceIdStr) throws Exception { ObjectNode rpcActualResultCancelAll = sendRpcObserve("ObserveCancelAll", null, deviceIdStr); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultCancelAll.get("result").asText()); awaitObserveReadAll(0, deviceIdStr); } - protected void awaitObserveReadAll(int cntObserve, String deviceIdStr) throws Exception { + protected void awaitObserveReadAll(int cntObserve, String deviceIdStr) throws Exception { await("ObserveReadAll after start client/test: countObserve " + cntObserve) .atMost(40, TimeUnit.SECONDS) .until(() -> cntObserve == getCntObserveAll(deviceIdStr)); } - protected void awaitUpdateRegistrationSuccess(LwM2MTestClient lwM2MTestClient, int cntUpdate) throws Exception { + + protected void awaitUpdateRegistrationSuccess(LwM2MTestClient lwM2MTestClient, int cntUpdate) throws Exception { cntUpdate = cntUpdate + lwM2MTestClient.getCountUpdateRegistrationSuccess(); int finalCntUpdate = cntUpdate; await("Update Registration client: countUpdateSuccess " + finalCntUpdate) .atMost(40, TimeUnit.SECONDS) .until(() -> finalCntUpdate <= lwM2MTestClient.getCountUpdateRegistrationSuccess()); } - protected void awaitObserveReadResource_3_0_9(int cntRead, String deviceIdStr) throws Exception { + + protected void awaitObserveReadResource_3_0_9(int cntRead, String deviceIdStr) throws Exception { await("Read value 3/0/9 after start observe: countRead " + cntRead) .atMost(40, TimeUnit.SECONDS) .until(() -> cntRead == getCntObserveAll(deviceIdStr)); @@ -380,19 +444,13 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { ObjectNode rpcActualResultBefore = sendRpcObserve("ObserveReadAll", null, deviceIdStr); assertEquals(ResponseCode.CONTENT.getName(), rpcActualResultBefore.get("result").asText()); JsonElement element = JsonParser.parseString(rpcActualResultBefore.get("value").asText()); - return element.isJsonArray() ? ((JsonArray)element).size() : null; + return element.isJsonArray() ? ((JsonArray) element).size() : null; } - private void sendRpcObserveWithContainsLwM2mSingleResource(String params, String deviceIdStr) throws Exception { + private void sendRpcObserveWithContainsLwM2mSingleResource(String params, String deviceIdStr, int cnt) throws Exception { String rpcActualResult = sendRpcObserveWithResultValue(params, deviceIdStr); assertTrue(rpcActualResult.contains("LwM2mSingleResource")); - assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceIdStr)).get()); - } - - private void sendRpcObserveCompositeWithContainsLwM2mSingleResource(String params, String deviceIdStr) throws Exception { - String rpcActualResult = sendRpcObserveCompositeWithResultValue(params, deviceIdStr); - assertTrue(rpcActualResult.contains("LwM2mSingleResource")); - assertEquals(Optional.of(1).get(), Optional.ofNullable(getCntObserveAll(deviceIdStr)).get()); + assertEquals(Optional.of(cnt).get(), Optional.ofNullable(getCntObserveAll(deviceIdStr)).get()); } private String sendRpcObserveWithResultValue(String params, String deviceIdStr) throws Exception { @@ -411,15 +469,25 @@ public class AbstractLwm2mClientTest extends AbstractContainerTest { String sendRpcRequest; if (params == null) { sendRpcRequest = "{\"method\": \"" + method + "\"}"; - } - else { + } else { sendRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"id\": \"" + params + "\"}}"; } return testRestClient.postRpcLwm2mParams(deviceIdStr, sendRpcRequest); } + protected ObjectNode sendRpcObserveComposite(String keys, String deviceIdStr) throws Exception { String method = "ObserveComposite"; String sendRpcRequest = "{\"method\": \"" + method + "\", \"params\": {\"keys\":" + keys + "}}"; return testRestClient.postRpcLwm2mParams(deviceIdStr, sendRpcRequest); } + + public void checkLatestTelemetryUploaded(DeviceId deviceId, long tsBefore, String... keyNames) throws Exception { + WsClient wsClient = subscribeToWebSocket(deviceId, "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); + wsClient.closeBlocking(); + Assert.assertEquals(actualLatestTelemetry.getData().size(), 1); + String actualKeyName = actualLatestTelemetry.getLatestValues().keySet().toArray()[0].toString(); + Assert.assertTrue(Sets.newHashSet(keyNames).contains(actualKeyName)); + Assert.assertTrue((Long) actualLatestTelemetry.getDataValuesByKey(actualKeyName).get(0) > tsBefore); + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2MTestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2MTestClient.java index 9fb9d4c67d..ad1294a637 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2MTestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/LwM2MTestClient.java @@ -72,6 +72,7 @@ import static org.eclipse.leshan.core.LwM2mId.DEVICE; import static org.eclipse.leshan.core.LwM2mId.FIRMWARE; import static org.eclipse.leshan.core.LwM2mId.SECURITY; import static org.eclipse.leshan.core.LwM2mId.SERVER; +import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.BINARY_APP_DATA_CONTAINER; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_FAILURE; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_STARTED; import static org.thingsboard.server.msa.connectivity.lwm2m.client.Lwm2mTestHelper.LwM2MClientState.ON_BOOTSTRAP_SUCCESS; @@ -107,6 +108,8 @@ public class LwM2MTestClient { private Set clientStates; private FwLwM2MDevice fwLwM2MDevice; + + private LwM2mBinaryAppDataContainer lwM2MBinaryAppDataContainer; private Map clientDtlsCid; private int countUpdateRegistrationSuccess; @@ -133,6 +136,8 @@ public class LwM2MTestClient { initializer.setInstancesForObject(DEVICE, lwM2MDevice = simpleLwM2MDevice); initializer.setClassForObject(ACCESS_CONTROL, DummyInstanceEnabler.class); initializer.setInstancesForObject(FIRMWARE, fwLwM2MDevice = new FwLwM2MDevice()); + initializer.setInstancesForObject(BINARY_APP_DATA_CONTAINER, lwM2MBinaryAppDataContainer = new LwM2mBinaryAppDataContainer(executor, 0), + new LwM2mBinaryAppDataContainer(executor, 1)); List enablers = initializer.createAll(); @@ -350,5 +355,8 @@ public class LwM2MTestClient { if (fwLwM2MDevice != null) { fwLwM2MDevice.destroy(); } + if (lwM2MBinaryAppDataContainer != null) { + lwM2MBinaryAppDataContainer.destroy(); + } } } \ No newline at end of file diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/Lwm2mTestHelper.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/Lwm2mTestHelper.java index 059876e94b..c8200342e2 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/Lwm2mTestHelper.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/client/Lwm2mTestHelper.java @@ -25,7 +25,7 @@ import static org.eclipse.leshan.client.object.Security.noSec; public class Lwm2mTestHelper { // Models - public static final String[] resources = new String[]{ "0.xml", "1.xml", "2.xml", "3.xml", "5.xml"}; + public static final String[] resources = new String[]{"0.xml", "1.xml", "2.xml", "3.xml", "5.xml", "19.xml"}; public static final int serverId = 1; public static final int port = 5685; @@ -44,19 +44,38 @@ public class Lwm2mTestHelper { public static final String CLIENT_PSK_IDENTITY = "SOME_PSK_ID"; public static final String CLIENT_PSK_KEY = "73656372657450534b73656372657450"; - public static String OBSERVE_ATTRIBUTES_WITH_PARAMS = + public static final int BINARY_APP_DATA_CONTAINER = 19; + + public static final String RESOURCE_ID_NAME_3_9 = "batteryLevel"; + public static final String RESOURCE_ID_NAME_3_14 = "UtfOffset"; + public static final String RESOURCE_ID_NAME_19_0_0 = "dataRead"; + public static final String RESOURCE_ID_NAME_19_0_2 = "dataCreationTime"; + public static final String RESOURCE_ID_NAME_19_1_0 = "dataWrite"; + + public static String OBSERVE_ATTRIBUTES_WITH_PARAMS = " {\n" + " \"keyName\": {\n" + - " \"/3_1.2/0/9\": \"batteryLevel\"\n" + + " \"/3_1.2/0/9\": \"batteryLevel\",\n" + + " \"/3_1.2/0/14\": \"UtfOffset\",\n" + + " \"/19_1.1/0/0\": \"dataRead\",\n" + + " \"/19_1.1/1/0\": \"dataWrite\",\n" + + " \"/19_1.1/0/2\": \"dataCreationTime\"\n" + " },\n" + " \"observe\": [\n" + - " \"/3_1.2/0/9\"\n" + + " \"/3_1.2/0/9\",\n" + + " \"/19_1.1/0/0\",\n" + + " \"/19_1.1/1/0\",\n" + + " \"/19_1.1/0/2\"\n" + " ],\n" + " \"attribute\": [\n" + + " \"/3_1.2/0/14\",\n" + + " \"/19_1.1/0/2\"\n" + " ],\n" + " \"telemetry\": [\n" + - " \"/3_1.2/0/9\"\n" + + " \"/3_1.2/0/9\",\n" + + " \"/19_1.1/0/0\",\n" + + " \"/19_1.1/1/0\"\n" + " ],\n" + " \"attributeLwm2m\": {}\n" + " }"; @@ -73,10 +92,6 @@ public class Lwm2mTestHelper { " \"clientOnlyObserveAfterConnect\": 1\n" + " }"; - public static final int BINARY_APP_DATA_CONTAINER = 19; - public static final int OBJECT_INSTANCE_ID_0 = 0; - public static final int OBJECT_INSTANCE_ID_1 = 1; - public enum LwM2MClientState { ON_INIT(0, "onInit"), @@ -97,8 +112,8 @@ public class Lwm2mTestHelper { ON_DEREGISTRATION_FAILURE(14, "onDeregistrationFailure"), ON_DEREGISTRATION_TIMEOUT(15, "onDeregistrationTimeout"), ON_EXPECTED_ERROR(16, "onUnexpectedError"), - ON_READ_CONNECTION_ID (17, "onReadConnection"), - ON_WRITE_CONNECTION_ID (18, "onWriteConnection"); + ON_READ_CONNECTION_ID(17, "onReadConnection"), + ON_WRITE_CONNECTION_ID(18, "onWriteConnection"); public int code; public String type; @@ -131,7 +146,7 @@ public class Lwm2mTestHelper { serverCoapConfig.setTransient(DTLS_CONNECTION_ID_LENGTH); serverCoapConfig.setTransient(DTLS_CONNECTION_ID_NODE_ID); serverCoapConfig.set(DTLS_CONNECTION_ID_LENGTH, cIdLength); - if ( cIdLength > 4) { + if (cIdLength > 4) { serverCoapConfig.set(DTLS_CONNECTION_ID_NODE_ID, 0); } else { serverCoapConfig.set(DTLS_CONNECTION_ID_NODE_ID, null); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java index 3eab4560fc..117e3c5b96 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveCompositeTest.java @@ -47,6 +47,6 @@ public class Lwm2mObserveCompositeTest extends AbstractLwm2mClientTest { @Test public void testObserveResource_Update_AfterUpdateRegistration() throws Exception { createLwm2mDevicesForConnectNoSec( name + "-" + RandomStringUtils.randomAlphanumeric(7), this.lwm2mDevicesForTest ); - observeCompositeResource_Update_AfterUpdateRegistration_test(this.lwm2mDevicesForTest.getLwM2MTestClient(), this.lwm2mDevicesForTest.getLwM2MDeviceTest().getId().toString()); + observeCompositeResource_Update_AfterUpdateRegistration_test(this.lwm2mDevicesForTest.getLwM2MTestClient(), this.lwm2mDevicesForTest.getLwM2MDeviceTest().getId()); } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java index 1e10a692c8..07e8780aa0 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/lwm2m/rpc/Lwm2mObserveTest.java @@ -47,6 +47,6 @@ public class Lwm2mObserveTest extends AbstractLwm2mClientTest { @Test public void testObserveResource_Update_AfterUpdateRegistration() throws Exception { createLwm2mDevicesForConnectNoSec( name + "-" + RandomStringUtils.randomAlphanumeric(7), this.lwm2mDevicesForTest ); - observeResource_Update_AfterUpdateRegistration_test(this.lwm2mDevicesForTest.getLwM2MTestClient(), this.lwm2mDevicesForTest.getLwM2MDeviceTest().getId().toString()); + observeResource_Update_AfterUpdateRegistration_test(this.lwm2mDevicesForTest.getLwM2MTestClient(), this.lwm2mDevicesForTest.getLwM2MDeviceTest().getId()); } } From c217769e96e96d0fa30d90e0b1df0e34bde641c7 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 18 Sep 2024 12:54:47 +0300 Subject: [PATCH 32/32] UI: Fixed incorrect show widgets in layout --- .../home/components/dashboard/dashboard.component.html | 2 +- .../modules/home/components/dashboard/dashboard.component.ts | 4 ++++ .../src/app/modules/home/models/dashboard-component.models.ts | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html b/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html index 33c8d659f6..91719334df 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html @@ -66,7 +66,7 @@
- + { diff --git a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts index 0b7b122112..19838773cf 100644 --- a/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/dashboard-component.models.ts @@ -707,7 +707,7 @@ export class DashboardWidget implements GridsterItem, IDashboardWidget { } set x(x: number) { - if (!this.dashboard.isMobileSize) { + if (!this.dashboard.isMobileSize && this.dashboard.isEdit) { if (this.widgetLayout) { this.widgetLayout.col = x; } else { @@ -728,7 +728,7 @@ export class DashboardWidget implements GridsterItem, IDashboardWidget { } set y(y: number) { - if (!this.dashboard.isMobileSize) { + if (!this.dashboard.isMobileSize && this.dashboard.isEdit) { if (this.widgetLayout) { this.widgetLayout.row = y; } else {