Browse Source

Merge remote-tracking branch 'upstream/master' into feature/edge-alarm-comment-support

pull/9968/head
Andrii Landiak 3 years ago
parent
commit
28e9aae2e5
  1. 2
      application/src/main/data/json/system/widget_types/bar_chart_with_labels.json
  2. 7
      application/src/main/data/upgrade/3.6.2/schema_update.sql
  3. 2
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  4. 16
      application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java
  5. 9
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  6. 21
      application/src/main/java/org/thingsboard/server/controller/TelemetryController.java
  7. 8
      application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java
  8. 1
      application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java
  9. 48
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  10. 15
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  11. 8
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  12. 7
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java
  13. 1
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
  14. 3
      application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/EntityHistoryCmd.java
  15. 18
      application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java
  16. 3
      application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/TimeSeriesCmd.java
  17. 6
      application/src/main/java/org/thingsboard/server/utils/TbNodeUpgradeUtils.java
  18. 14
      application/src/main/resources/thingsboard.yml
  19. 61
      application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java
  20. 75
      application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java
  21. 5
      application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java
  22. 1
      common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java
  23. 92
      common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java
  24. 26
      common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java
  25. 22
      common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java
  26. 12
      common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java
  27. 7
      common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentDescriptor.java
  28. 6
      common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java
  29. 2
      common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/DefaultCoapClientContext.java
  30. 10
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
  31. 2
      common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java
  32. 2
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java
  33. 187
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/AbstractActivityManager.java
  34. 26
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/ActivityManager.java
  35. 24
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/ActivityReportCallback.java
  36. 12
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/ActivityState.java
  37. 24
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/ActivityStrategy.java
  38. 47
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/ActivityStrategyType.java
  39. 30
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/AllEventsActivityStrategy.java
  40. 40
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/FirstAndLastEventActivityStrategy.java
  41. 40
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/FirstEventActivityStrategy.java
  42. 39
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/LastEventActivityStrategy.java
  43. 124
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  44. 149
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportActivityManager.java
  45. 44
      common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/ActivityStrategyTypeTest.java
  46. 42
      common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/AllEventsActivityStrategyTest.java
  47. 53
      common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/FirstAndLastEventActivityStrategyTest.java
  48. 52
      common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/FirstEventActivityStrategyTest.java
  49. 43
      common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/LastEventActivityStrategyTest.java
  50. 508
      common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/service/TransportActivityManagerTest.java
  51. 3
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityConfiguration.java
  52. 2
      dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
  53. 5
      dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java
  54. 5
      dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java
  55. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java
  56. 4
      dao/src/main/java/org/thingsboard/server/dao/sql/component/SqlComponentDescriptorInsertRepository.java
  57. 23
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
  58. 24
      dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java
  59. 18
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  60. 2
      dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java
  61. 45
      dao/src/main/java/org/thingsboard/server/dao/util/TimeUtils.java
  62. 4
      dao/src/main/resources/sql/schema-entities.sql
  63. 60
      dao/src/test/java/org/thingsboard/server/dao/util/TimeUtilsTest.java
  64. 4
      monitoring/src/main/java/org/thingsboard/monitoring/config/MonitoringTarget.java
  65. 7
      monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java
  66. 6
      monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java
  67. 9
      monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportType.java
  68. 2
      monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java
  69. 2
      monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotification.java
  70. 2
      monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceRecoveryNotification.java
  71. 18
      monitoring/src/main/java/org/thingsboard/monitoring/notification/NotificationService.java
  72. 4
      monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/NotificationChannel.java
  73. 11
      monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackNotificationChannel.java
  74. 96
      monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java
  75. 13
      monitoring/src/main/resources/tb-monitoring.yml
  76. 2
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java
  77. 9
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java
  78. 41
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java
  79. 204
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/rule/node/MqttNodeTest.java
  80. 79
      msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json
  81. 25
      msa/black-box-tests/src/test/resources/docker-compose.mosquitto.yml
  82. 18
      msa/black-box-tests/src/test/resources/mosquitto/mosquitto.conf
  83. 179
      rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java
  84. 2
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleNode.java
  85. 2
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java
  86. 1
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/util/TbNodeUtils.java
  87. 30
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java
  88. 1
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNodeConfiguration.java
  89. 27
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java
  90. 1
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNodeConfiguration.java
  91. 31
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java
  92. 50
      rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js
  93. 52
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/AbstractRuleNodeUpgradeTest.java
  94. 53
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNodeTest.java
  95. 55
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeTest.java
  96. 25
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java
  97. 67
      rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbMsgDeduplicationNodeTest.java
  98. 1
      ui-ngx/package.json
  99. 161
      ui-ngx/src/app/core/api/data-aggregator.ts
  100. 96
      ui-ngx/src/app/core/api/entity-data-subscription.ts

2
application/src/main/data/json/system/widget_types/bar_chart_with_labels.json

@ -20,7 +20,7 @@
"latestDataKeySettingsDirective": "", "latestDataKeySettingsDirective": "",
"hasBasicMode": true, "hasBasicMode": true,
"basicModeDirective": "tb-bar-chart-with-labels-basic-config", "basicModeDirective": "tb-bar-chart-with-labels-basic-config",
"defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Humidity\",\"color\":\"rgb(125, 142, 255)\",\"settings\":{},\"_hash\":0.8587686344902596,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 50) {\\n\\tvalue = 50;\\n} else if (value > 80) {\\n\\tvalue = 80;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Soil moisture\",\"color\":\"rgb(249, 111, 255)\",\"settings\":{},\"_hash\":0.9111685461089025,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 30) {\\n\\tvalue = 30;\\n} else if (value > 90) {\\n\\tvalue = 90;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Leaf wetness\",\"color\":\"rgb(255, 163, 137)\",\"settings\":{},\"_hash\":0.8487533373085416,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 40) {\\n\\tvalue = 40;\\n} else if (value > 70) {\\n\\tvalue = 70;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Cloud cover\",\"color\":\"#FFED53\",\"settings\":{},\"_hash\":0.7690144858984289,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 20) {\\n\\tvalue = 20;\\n} else if (value > 60) {\\n\\tvalue = 60;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null}],\"alarmFilterConfig\":{\"statusList\":[\"ACTIVE\"]}}],\"timewindow\":{\"hideInterval\":false,\"hideLastInterval\":false,\"hideQuickInterval\":false,\"hideAggregation\":false,\"hideAggInterval\":false,\"hideTimezone\":false,\"selectedTab\":1,\"history\":{\"historyType\":2,\"timewindowMs\":60000,\"interval\":2592000000,\"fixedTimewindow\":{\"startTimeMs\":1704293713163,\"endTimeMs\":1704380113163},\"quickInterval\":\"CURRENT_HALF_YEAR\"},\"aggregation\":{\"type\":\"AVG\",\"limit\":25000},\"timezone\":null},\"showTitle\":true,\"backgroundColor\":\"rgba(0, 0, 0, 0)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"0px\",\"settings\":{\"showBarLabel\":true,\"barLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"12px\"},\"barLabelColor\":\"rgba(0, 0, 0, 0.54)\",\"showBarValue\":true,\"barValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"700\",\"lineHeight\":\"12px\"},\"barValueColor\":\"rgba(0, 0, 0, 0.76)\",\"showLegend\":true,\"legendPosition\":\"top\",\"legendLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"legendLabelColor\":\"rgba(0, 0, 0, 0.76)\",\"showTooltip\":true,\"tooltipValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"500\",\"lineHeight\":\"16px\"},\"tooltipValueColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipShowDate\":true,\"tooltipDateFormat\":{\"format\":\"MMMM y\",\"lastUpdateAgo\":false,\"custom\":true},\"tooltipDateFont\":{\"family\":\"Roboto\",\"size\":11,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"tooltipDateColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipBackgroundColor\":\"rgba(255, 255, 255, 0.76)\",\"tooltipBackgroundBlur\":4,\"background\":{\"type\":\"color\",\"color\":\"#fff\",\"overlay\":{\"enabled\":false,\"color\":\"rgba(255,255,255,0.72)\",\"blur\":3}}},\"title\":\"Bar chart with labels\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":null,\"mobileHeight\":null,\"configMode\":\"basic\",\"actions\":{},\"showTitleIcon\":false,\"titleIcon\":\"public\",\"iconColor\":\"#1F6BDD\",\"useDashboardTimewindow\":false,\"displayTimewindow\":true,\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"lineHeight\":\"24px\"},\"titleColor\":\"rgba(0, 0, 0, 0.87)\",\"titleTooltip\":\"\",\"widgetStyle\":{},\"widgetCss\":\"\",\"pageSize\":1024,\"units\":\"%\",\"decimals\":0,\"noDataDisplayMessage\":\"\",\"timewindowStyle\":{\"showIcon\":false,\"iconSize\":\"24px\",\"icon\":null,\"iconPosition\":\"left\",\"font\":{\"size\":12,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"400\",\"style\":\"normal\",\"lineHeight\":\"16px\"},\"color\":\"rgba(0, 0, 0, 0.38)\",\"displayTypePrefix\":true},\"margin\":\"0px\",\"borderRadius\":\"0px\",\"iconSize\":\"24px\"}" "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Humidity\",\"color\":\"rgb(125, 142, 255)\",\"settings\":{},\"_hash\":0.8587686344902596,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 50) {\\n\\tvalue = 50;\\n} else if (value > 80) {\\n\\tvalue = 80;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Soil moisture\",\"color\":\"rgb(249, 111, 255)\",\"settings\":{},\"_hash\":0.9111685461089025,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 30) {\\n\\tvalue = 30;\\n} else if (value > 90) {\\n\\tvalue = 90;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Leaf wetness\",\"color\":\"rgb(255, 163, 137)\",\"settings\":{},\"_hash\":0.8487533373085416,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 40) {\\n\\tvalue = 40;\\n} else if (value > 70) {\\n\\tvalue = 70;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Cloud cover\",\"color\":\"#FFED53\",\"settings\":{},\"_hash\":0.7690144858984289,\"funcBody\":\"var value = prevValue + Math.random() * 40 - 20;\\nif (value < 20) {\\n\\tvalue = 20;\\n} else if (value > 60) {\\n\\tvalue = 60;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null}],\"alarmFilterConfig\":{\"statusList\":[\"ACTIVE\"]}}],\"timewindow\":{\"hideInterval\":false,\"hideLastInterval\":false,\"hideQuickInterval\":false,\"hideAggregation\":false,\"hideAggInterval\":false,\"hideTimezone\":false,\"selectedTab\":1,\"history\":{\"historyType\":2,\"timewindowMs\":60000,\"interval\":\"MONTH\",\"fixedTimewindow\":{\"startTimeMs\":1704293713163,\"endTimeMs\":1704380113163},\"quickInterval\":\"CURRENT_HALF_YEAR\"},\"aggregation\":{\"type\":\"AVG\",\"limit\":25000},\"timezone\":null},\"showTitle\":true,\"backgroundColor\":\"rgba(0, 0, 0, 0)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"0px\",\"settings\":{\"showBarLabel\":true,\"barLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"12px\"},\"barLabelColor\":\"rgba(0, 0, 0, 0.54)\",\"showBarValue\":true,\"barValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"700\",\"lineHeight\":\"12px\"},\"barValueColor\":\"rgba(0, 0, 0, 0.76)\",\"showLegend\":true,\"legendPosition\":\"top\",\"legendLabelFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"legendLabelColor\":\"rgba(0, 0, 0, 0.76)\",\"showTooltip\":true,\"tooltipValueFont\":{\"family\":\"Roboto\",\"size\":12,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"500\",\"lineHeight\":\"16px\"},\"tooltipValueColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipShowDate\":true,\"tooltipDateFormat\":{\"format\":\"MMMM y\",\"lastUpdateAgo\":false,\"custom\":true},\"tooltipDateFont\":{\"family\":\"Roboto\",\"size\":11,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"400\",\"lineHeight\":\"16px\"},\"tooltipDateColor\":\"rgba(0, 0, 0, 0.76)\",\"tooltipBackgroundColor\":\"rgba(255, 255, 255, 0.76)\",\"tooltipBackgroundBlur\":4,\"background\":{\"type\":\"color\",\"color\":\"#fff\",\"overlay\":{\"enabled\":false,\"color\":\"rgba(255,255,255,0.72)\",\"blur\":3}}},\"title\":\"Bar chart with labels\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":null,\"mobileHeight\":null,\"configMode\":\"basic\",\"actions\":{},\"showTitleIcon\":false,\"titleIcon\":\"public\",\"iconColor\":\"#1F6BDD\",\"useDashboardTimewindow\":false,\"displayTimewindow\":true,\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"lineHeight\":\"24px\"},\"titleColor\":\"rgba(0, 0, 0, 0.87)\",\"titleTooltip\":\"\",\"widgetStyle\":{},\"widgetCss\":\"\",\"pageSize\":1024,\"units\":\"%\",\"decimals\":0,\"noDataDisplayMessage\":\"\",\"timewindowStyle\":{\"showIcon\":false,\"iconSize\":\"24px\",\"icon\":null,\"iconPosition\":\"left\",\"font\":{\"size\":12,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"400\",\"style\":\"normal\",\"lineHeight\":\"16px\"},\"color\":\"rgba(0, 0, 0, 0.38)\",\"displayTypePrefix\":true},\"margin\":\"0px\",\"borderRadius\":\"0px\",\"iconSize\":\"24px\"}"
}, },
"tags": [ "tags": [
"bar chart", "bar chart",

7
application/src/main/data/upgrade/3.6.2/schema_update.sql

@ -21,3 +21,10 @@ DROP INDEX IF EXISTS idx_rule_node_type_configuration_version;
CREATE INDEX IF NOT EXISTS idx_rule_node_type_id_configuration_version ON rule_node(type, id, configuration_version); CREATE INDEX IF NOT EXISTS idx_rule_node_type_id_configuration_version ON rule_node(type, id, configuration_version);
-- RULE NODE INDEXES UPDATE END -- RULE NODE INDEXES UPDATE END
-- RULE NODE QUEUE UPDATE START
ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS queue_name varchar(255);
ALTER TABLE component_descriptor ADD COLUMN IF NOT EXISTS has_queue_name boolean DEFAULT false;
-- RULE NODE QUEUE UPDATE END

2
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -29,7 +29,6 @@ import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider;
import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.SmsService;
@ -101,6 +100,7 @@ import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService;
import org.thingsboard.server.service.executors.NotificationExecutorService; import org.thingsboard.server.service.executors.NotificationExecutorService;
import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider;
import org.thingsboard.server.service.executors.SharedEventLoopGroupService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService;
import org.thingsboard.server.service.mail.MailExecutorService; import org.thingsboard.server.service.mail.MailExecutorService;
import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache;

16
application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java

@ -25,10 +25,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.LinkedHashMapRemoveEldest; import org.thingsboard.common.util.LinkedHashMapRemoveEldest;
import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor;
@ -61,7 +57,14 @@ import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg;
import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg; import org.thingsboard.server.common.msg.timeout.DeviceActorServerSideRpcTimeoutMsg;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
@ -87,10 +90,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportUpdateCredentialsProto; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportUpdateCredentialsProto;
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg;
import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg;
import org.thingsboard.server.service.rpc.RpcSubmitStrategy; import org.thingsboard.server.service.rpc.RpcSubmitStrategy;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -173,7 +173,7 @@ public class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcesso
private EdgeId findRelatedEdgeId() { private EdgeId findRelatedEdgeId() {
List<EntityRelation> result = List<EntityRelation> result =
systemContext.getRelationService().findByToAndType(tenantId, deviceId, EntityRelation.EDGE_TYPE, RelationTypeGroup.COMMON); systemContext.getRelationService().findByToAndType(tenantId, deviceId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON);
if (result != null && result.size() > 0) { if (result != null && result.size() > 0) {
EntityRelation relationToEdge = result.get(0); EntityRelation relationToEdge = result.get(0);
if (relationToEdge.getFrom() != null && relationToEdge.getFrom().getId() != null) { if (relationToEdge.getFrom() != null && relationToEdge.getFrom().getId() != null) {

9
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -186,7 +186,7 @@ class DefaultTbContext implements TbContext {
@Override @Override
public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer<Throwable> onFailure) { public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), tbMsg.getOriginator()); TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), tbMsg.getOriginator());
enqueue(tpi, tbMsg, onFailure, onSuccess); enqueue(tpi, tbMsg, onFailure, onSuccess);
} }
@ -311,7 +311,7 @@ class DefaultTbContext implements TbContext {
@Override @Override
public boolean isLocalEntity(EntityId entityId) { public boolean isLocalEntity(EntityId entityId) {
return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getTenantId(), entityId).isMyPartition(); return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), entityId).isMyPartition();
} }
private void scheduleMsgWithDelay(TbActorMsg msg, long delayInMs, TbActorRef target) { private void scheduleMsgWithDelay(TbActorMsg msg, long delayInMs, TbActorRef target) {
@ -510,6 +510,11 @@ class DefaultTbContext implements TbContext {
return ruleChainName; return ruleChainName;
} }
@Override
public String getQueueName() {
return getSelf().getQueueName();
}
@Override @Override
public TenantId getTenantId() { public TenantId getTenantId() {
return nodeCtx.getTenantId(); return nodeCtx.getTenantId();

21
application/src/main/java/org/thingsboard/server/controller/TelemetryController.java

@ -45,6 +45,8 @@ import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult; import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.kv.AggregationParams;
import org.thingsboard.server.common.data.kv.IntervalType;
import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
@ -310,8 +312,12 @@ public class TelemetryController extends BaseController {
@RequestParam(name = "startTs") Long startTs, @RequestParam(name = "startTs") Long startTs,
@ApiParam(value = "A long value representing the end timestamp of the time range in milliseconds, UTC.") @ApiParam(value = "A long value representing the end timestamp of the time range in milliseconds, UTC.")
@RequestParam(name = "endTs") Long endTs, @RequestParam(name = "endTs") Long endTs,
@ApiParam(value = "A string value representing the type fo the interval.", allowableValues = "MILLISECONDS, WEEK, WEEK_ISO, MONTH, QUARTER")
@RequestParam(name = "intervalType", required = false) IntervalType intervalType,
@ApiParam(value = "A long value representing the aggregation interval range in milliseconds.") @ApiParam(value = "A long value representing the aggregation interval range in milliseconds.")
@RequestParam(name = "interval", defaultValue = "0") Long interval, @RequestParam(name = "interval", defaultValue = "0") Long interval,
@ApiParam(value = "A string value representing the timezone that will be used to calculate exact timestamps for 'WEEK', 'WEEK_ISO', 'MONTH' and 'QUARTER' interval types.")
@RequestParam(name = "timeZone", required = false) String timeZone,
@ApiParam(value = "An integer value that represents a max number of timeseries data points to fetch." + @ApiParam(value = "An integer value that represents a max number of timeseries data points to fetch." +
" This parameter is used only in the case if 'agg' parameter is set to 'NONE'.", defaultValue = "100") " This parameter is used only in the case if 'agg' parameter is set to 'NONE'.", defaultValue = "100")
@RequestParam(name = "limit", defaultValue = "100") Integer limit, @RequestParam(name = "limit", defaultValue = "100") Integer limit,
@ -325,11 +331,16 @@ public class TelemetryController extends BaseController {
@RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException {
return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr,
(result, tenantId, entityId) -> { (result, tenantId, entityId) -> {
// If interval is 0, convert this to a NONE aggregation, which is probably what the user really wanted AggregationParams params;
Aggregation agg = interval == 0L ? Aggregation.valueOf(Aggregation.NONE.name()) : Aggregation.valueOf(aggStr); Aggregation agg = Aggregation.valueOf(aggStr);
List<ReadTsKvQuery> queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg, orderBy)) if (Aggregation.NONE.equals(agg)) {
.collect(Collectors.toList()); params = AggregationParams.none();
} else if (intervalType == null || IntervalType.MILLISECONDS.equals(intervalType)) {
params = interval == 0L ? AggregationParams.none() : AggregationParams.milliseconds(agg, interval);
} else {
params = AggregationParams.calendar(agg, intervalType, timeZone);
}
List<ReadTsKvQuery> queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, params, limit, orderBy)).collect(Collectors.toList());
Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor());
}); });
} }

8
application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java

@ -209,6 +209,7 @@ public class WidgetTypeController extends AutoCommitController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/widgetTypes", params = {"isSystem", "bundleAlias"}, method = RequestMethod.GET) @RequestMapping(value = "/widgetTypes", params = {"isSystem", "bundleAlias"}, method = RequestMethod.GET)
@ResponseBody @ResponseBody
@Deprecated
public List<WidgetType> getBundleWidgetTypesByBundleAlias( public List<WidgetType> getBundleWidgetTypesByBundleAlias(
@ApiParam(value = "System or Tenant", required = true) @ApiParam(value = "System or Tenant", required = true)
@RequestParam boolean isSystem, @RequestParam boolean isSystem,
@ -241,6 +242,7 @@ public class WidgetTypeController extends AutoCommitController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/widgetTypesDetails", params = {"isSystem", "bundleAlias"}, method = RequestMethod.GET) @RequestMapping(value = "/widgetTypesDetails", params = {"isSystem", "bundleAlias"}, method = RequestMethod.GET)
@ResponseBody @ResponseBody
@Deprecated
public List<WidgetTypeDetails> getBundleWidgetTypesDetailsByBundleAlias( public List<WidgetTypeDetails> getBundleWidgetTypesDetailsByBundleAlias(
@ApiParam(value = "System or Tenant", required = true) @ApiParam(value = "System or Tenant", required = true)
@RequestParam boolean isSystem, @RequestParam boolean isSystem,
@ -256,7 +258,7 @@ public class WidgetTypeController extends AutoCommitController {
return checkNotNull(widgetTypeService.findWidgetTypesDetailsByWidgetsBundleId(getTenantId(), widgetsBundle.getId())); return checkNotNull(widgetTypeService.findWidgetTypesDetailsByWidgetsBundleId(getTenantId(), widgetsBundle.getId()));
} }
@ApiOperation(value = "Get all Widget types details for specified Bundle (getBundleWidgetTypes)", @ApiOperation(value = "Get all Widget types details for specified Bundle (getBundleWidgetTypesDetails)",
notes = "Returns an array of Widget Type Details objects that belong to specified Widget Bundle." + WIDGET_TYPE_DETAILS_DESCRIPTION + " " + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) notes = "Returns an array of Widget Type Details objects that belong to specified Widget Bundle." + WIDGET_TYPE_DETAILS_DESCRIPTION + " " + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/widgetTypesDetails", params = {"widgetsBundleId"}, method = RequestMethod.GET) @RequestMapping(value = "/widgetTypesDetails", params = {"widgetsBundleId"}, method = RequestMethod.GET)
@ -292,6 +294,7 @@ public class WidgetTypeController extends AutoCommitController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/widgetTypesInfos", params = {"isSystem", "bundleAlias"}, method = RequestMethod.GET) @RequestMapping(value = "/widgetTypesInfos", params = {"isSystem", "bundleAlias"}, method = RequestMethod.GET)
@ResponseBody @ResponseBody
@Deprecated
public List<WidgetTypeInfo> getBundleWidgetTypesInfosByBundleAlias( public List<WidgetTypeInfo> getBundleWidgetTypesInfosByBundleAlias(
@ApiParam(value = "System or Tenant", required = true) @ApiParam(value = "System or Tenant", required = true)
@RequestParam boolean isSystem, @RequestParam boolean isSystem,
@ -345,6 +348,7 @@ public class WidgetTypeController extends AutoCommitController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/widgetType", params = {"isSystem", "bundleAlias", "alias"}, method = RequestMethod.GET) @RequestMapping(value = "/widgetType", params = {"isSystem", "bundleAlias", "alias"}, method = RequestMethod.GET)
@ResponseBody @ResponseBody
@Deprecated
public WidgetType getWidgetTypeByBundleAliasAndTypeAlias( public WidgetType getWidgetTypeByBundleAliasAndTypeAlias(
@ApiParam(value = "System or Tenant", required = true) @ApiParam(value = "System or Tenant", required = true)
@RequestParam boolean isSystem, @RequestParam boolean isSystem,
@ -365,7 +369,7 @@ public class WidgetTypeController extends AutoCommitController {
} }
@ApiOperation(value = "Get Widget Type (getWidgetType)", @ApiOperation(value = "Get Widget Type (getWidgetType)",
notes = "Get the Widget Type by FQN. " + WIDGET_TYPE_DESCRIPTION + AVAILABLE_FOR_ANY_AUTHORIZED_USER) notes = "Get the Widget Type by FQN. " + WIDGET_TYPE_DESCRIPTION + AVAILABLE_FOR_ANY_AUTHORIZED_USER, hidden = true)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/widgetType", params = {"fqn"}, method = RequestMethod.GET) @RequestMapping(value = "/widgetType", params = {"fqn"}, method = RequestMethod.GET)
@ResponseBody @ResponseBody

1
application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java

@ -189,6 +189,7 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe
scannedComponent.setName(ruleNodeAnnotation.name()); scannedComponent.setName(ruleNodeAnnotation.name());
scannedComponent.setScope(ruleNodeAnnotation.scope()); scannedComponent.setScope(ruleNodeAnnotation.scope());
scannedComponent.setClusteringMode(ruleNodeAnnotation.clusteringMode()); scannedComponent.setClusteringMode(ruleNodeAnnotation.clusteringMode());
scannedComponent.setHasQueueName(ruleNodeAnnotation.hasQueueName());
NodeDefinition nodeDefinition = prepareNodeDefinition(clazz, ruleNodeAnnotation); NodeDefinition nodeDefinition = prepareNodeDefinition(clazz, ruleNodeAnnotation);
ObjectNode configurationDescriptor = JacksonUtil.newObjectNode(); ObjectNode configurationDescriptor = JacksonUtil.newObjectNode();
JsonNode node = JacksonUtil.valueToTree(nodeDefinition); JsonNode node = JacksonUtil.valueToTree(nodeDefinition);

48
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -85,6 +85,7 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration;
import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.DeviceService;
@ -169,6 +170,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Autowired @Autowired
private TimeseriesService tsService; private TimeseriesService tsService;
@Autowired
private DeviceConnectivityConfiguration connectivityConfiguration;
@Value("${state.persistToTelemetry:false}") @Value("${state.persistToTelemetry:false}")
@Getter @Getter
private boolean persistActivityToTelemetry; private boolean persistActivityToTelemetry;
@ -260,7 +264,6 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
ObjectNode node = JacksonUtil.newObjectNode(); ObjectNode node = JacksonUtil.newObjectNode();
node.put("baseUrl", "http://localhost:8080"); node.put("baseUrl", "http://localhost:8080");
node.put("prohibitDifferentUrl", false); node.put("prohibitDifferentUrl", false);
node.set("connectivity", createDeviceConnectivityConfiguration());
generalSettings.setJsonValue(node); generalSettings.setJsonValue(node);
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings);
@ -285,51 +288,10 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
AdminSettings connectivitySettings = new AdminSettings(); AdminSettings connectivitySettings = new AdminSettings();
connectivitySettings.setTenantId(TenantId.SYS_TENANT_ID); connectivitySettings.setTenantId(TenantId.SYS_TENANT_ID);
connectivitySettings.setKey("connectivity"); connectivitySettings.setKey("connectivity");
connectivitySettings.setJsonValue(createDeviceConnectivityConfiguration()); connectivitySettings.setJsonValue(JacksonUtil.valueToTree(connectivityConfiguration.getConnectivity()));
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, connectivitySettings); adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, connectivitySettings);
} }
private ObjectNode createDeviceConnectivityConfiguration() {
ObjectNode config = JacksonUtil.newObjectNode();
ObjectNode http = JacksonUtil.newObjectNode();
http.put("enabled", true);
http.put("host", "");
http.put("port", 8080);
config.set("http", http);
ObjectNode https = JacksonUtil.newObjectNode();
https.put("enabled", false);
https.put("host", "");
https.put("port", 443);
config.set("https", https);
ObjectNode mqtt = JacksonUtil.newObjectNode();
mqtt.put("enabled", true);
mqtt.put("host", "");
mqtt.put("port", 1883);
config.set("mqtt", mqtt);
ObjectNode mqtts = JacksonUtil.newObjectNode();
mqtts.put("enabled", false);
mqtts.put("host", "");
mqtts.put("port", 8883);
config.set("mqtts", mqtts);
ObjectNode coap = JacksonUtil.newObjectNode();
coap.put("enabled", true);
coap.put("host", "");
coap.put("port", 5683);
config.set("coap", coap);
ObjectNode coaps = JacksonUtil.newObjectNode();
coaps.put("enabled", false);
coaps.put("host", "");
coaps.put("port", 5684);
config.set("coaps", coaps);
return config;
}
@Override @Override
public void createRandomJwtSettings() throws Exception { public void createRandomJwtSettings() throws Exception {
jwtSettingsService.createRandomJwtSettings(); jwtSettingsService.createRandomJwtSettings();

15
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -44,6 +44,8 @@ import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
@ -57,6 +59,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
@ -68,8 +71,8 @@ import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService;
@ -77,6 +80,7 @@ import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -116,6 +120,7 @@ public class DefaultTbClusterService implements TbClusterService {
private final TbDeviceProfileCache deviceProfileCache; private final TbDeviceProfileCache deviceProfileCache;
private final TbAssetProfileCache assetProfileCache; private final TbAssetProfileCache assetProfileCache;
private final GatewayNotificationsService gatewayNotificationsService; private final GatewayNotificationsService gatewayNotificationsService;
private final EdgeService edgeService;
@Override @Override
public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) {
@ -546,11 +551,17 @@ public class DefaultTbClusterService implements TbClusterService {
pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null);
break; break;
case UNASSIGNED_FROM_EDGE: case UNASSIGNED_FROM_EDGE:
pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), null), null); EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId);
pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null);
break; break;
} }
} }
private EdgeId findRelatedEdgeIdIfAny(TenantId tenantId, EntityId entityId) {
PageData<EdgeId> pageData = edgeService.findRelatedEdgeIdsByEntityId(tenantId, entityId, new PageLink(1));
return Optional.ofNullable(pageData).filter(pd -> pd.getTotalElements() > 0).map(pd -> pd.getData().get(0)).orElse(null);
}
@Override @Override
public void onQueueChange(Queue queue) { public void onQueueChange(Queue queue) {
log.trace("[{}][{}] Processing queue change [{}] event", queue.getTenantId(), queue.getId(), queue.getName()); log.trace("[{}][{}] Processing queue change [{}] event", queue.getTenantId(), queue.getId(), queue.getName());

8
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -185,6 +185,10 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
@Getter @Getter
private int initFetchPackSize; private int initFetchPackSize;
@Value("${state.telemetryTtl:0}")
@Getter
private int telemetryTtl;
private ListeningExecutorService deviceStateExecutor; private ListeningExecutorService deviceStateExecutor;
final ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>(); final ConcurrentMap<DeviceId, DeviceStateData> deviceStates = new ConcurrentHashMap<>();
@ -803,7 +807,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
tsSubService.saveAndNotifyInternal( tsSubService.saveAndNotifyInternal(
TenantId.SYS_TENANT_ID, deviceId, TenantId.SYS_TENANT_ID, deviceId,
Collections.singletonList(new BasicTsKvEntry(getCurrentTimeMillis(), new LongDataEntry(key, value))), Collections.singletonList(new BasicTsKvEntry(getCurrentTimeMillis(), new LongDataEntry(key, value))),
new TelemetrySaveCallback<>(deviceId, key, value)); telemetryTtl, new TelemetrySaveCallback<>(deviceId, key, value));
} else { } else {
tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value)); tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value));
} }
@ -814,7 +818,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService<Dev
tsSubService.saveAndNotifyInternal( tsSubService.saveAndNotifyInternal(
TenantId.SYS_TENANT_ID, deviceId, TenantId.SYS_TENANT_ID, deviceId,
Collections.singletonList(new BasicTsKvEntry(getCurrentTimeMillis(), new BooleanDataEntry(key, value))), Collections.singletonList(new BasicTsKvEntry(getCurrentTimeMillis(), new BooleanDataEntry(key, value))),
new TelemetrySaveCallback<>(deviceId, key, value)); telemetryTtl, new TelemetrySaveCallback<>(deviceId, key, value));
} else { } else {
tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value)); tsSubService.saveAttrAndNotify(TenantId.SYS_TENANT_ID, deviceId, SERVER_SCOPE, key, value, new TelemetrySaveCallback<>(deviceId, key, value));
} }

7
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java

@ -560,17 +560,14 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
List<String> keys = cmd.getKeys(); List<String> keys = cmd.getKeys();
List<ReadTsKvQuery> finalTsKvQueryList; List<ReadTsKvQuery> finalTsKvQueryList;
List<ReadTsKvQuery> tsKvQueryList = keys.stream().map(key -> { List<ReadTsKvQuery> tsKvQueryList = keys.stream().map(key -> {
var query = new BaseReadTsKvQuery( var query = new BaseReadTsKvQuery(key, cmd.getStartTs(), cmd.getEndTs(), cmd.toAggregationParams(), getLimit(cmd.getLimit()));
key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg()
);
queriesKeys.put(query.getId(), query.getKey()); queriesKeys.put(query.getId(), query.getKey());
return query; return query;
}).collect(Collectors.toList()); }).collect(Collectors.toList());
if (cmd.isFetchLatestPreviousPoint()) { if (cmd.isFetchLatestPreviousPoint()) {
finalTsKvQueryList = new ArrayList<>(tsKvQueryList); finalTsKvQueryList = new ArrayList<>(tsKvQueryList);
finalTsKvQueryList.addAll(keys.stream().map(key -> { finalTsKvQueryList.addAll(keys.stream().map(key -> {
var query = new BaseReadTsKvQuery( var query = new BaseReadTsKvQuery(key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.toAggregationParams(), 1);
key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg());
queriesKeys.put(query.getId(), query.getKey()); queriesKeys.put(query.getId(), query.getKey());
return query; return query;
} }

1
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@ -21,7 +21,6 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;

3
application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/EntityHistoryCmd.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.ws.telemetry.cmd.v2;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.IntervalType;
import java.util.List; import java.util.List;
@ -26,7 +27,9 @@ public class EntityHistoryCmd implements GetTsCmd {
private List<String> keys; private List<String> keys;
private long startTs; private long startTs;
private long endTs; private long endTs;
private IntervalType intervalType;
private long interval; private long interval;
private String timeZoneId;
private int limit; private int limit;
private Aggregation agg; private Aggregation agg;
private boolean fetchLatestPreviousPoint; private boolean fetchLatestPreviousPoint;

18
application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/GetTsCmd.java

@ -16,6 +16,8 @@
package org.thingsboard.server.service.ws.telemetry.cmd.v2; package org.thingsboard.server.service.ws.telemetry.cmd.v2;
import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.AggregationParams;
import org.thingsboard.server.common.data.kv.IntervalType;
import java.util.List; import java.util.List;
@ -27,12 +29,28 @@ public interface GetTsCmd {
List<String> getKeys(); List<String> getKeys();
IntervalType getIntervalType();
long getInterval(); long getInterval();
String getTimeZoneId();
int getLimit(); int getLimit();
Aggregation getAgg(); Aggregation getAgg();
boolean isFetchLatestPreviousPoint(); boolean isFetchLatestPreviousPoint();
default AggregationParams toAggregationParams() {
var agg = getAgg();
var intervalType = getIntervalType();
if (agg == null || Aggregation.NONE.equals(agg)) {
return AggregationParams.none();
} else if (intervalType == null || IntervalType.MILLISECONDS.equals(intervalType)) {
return AggregationParams.milliseconds(agg, getInterval());
} else {
return AggregationParams.calendar(agg, intervalType, getTimeZoneId());
}
}
} }

3
application/src/main/java/org/thingsboard/server/service/ws/telemetry/cmd/v2/TimeSeriesCmd.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.ws.telemetry.cmd.v2;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.IntervalType;
import java.util.List; import java.util.List;
@ -27,7 +28,9 @@ public class TimeSeriesCmd implements GetTsCmd {
private List<String> keys; private List<String> keys;
private long startTs; private long startTs;
private long timeWindow; private long timeWindow;
private IntervalType intervalType;
private long interval; private long interval;
private String timeZoneId;
private int limit; private int limit;
private Aggregation agg; private Aggregation agg;
private boolean fetchLatestPreviousPoint; private boolean fetchLatestPreviousPoint;

6
application/src/main/java/org/thingsboard/server/utils/TbNodeUpgradeUtils.java

@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.data.util.TbPair;
import org.thingsboard.server.service.component.RuleNodeClassInfo; import org.thingsboard.server.service.component.RuleNodeClassInfo;
import static org.thingsboard.server.common.data.DataConstants.QUEUE_NAME;
@Slf4j @Slf4j
public class TbNodeUpgradeUtils { public class TbNodeUpgradeUtils {
@ -44,9 +46,13 @@ public class TbNodeUpgradeUtils {
} else { } else {
var tbVersionedNode = getTbVersionedNode(nodeInfo); var tbVersionedNode = getTbVersionedNode(nodeInfo);
try { try {
JsonNode queueName = oldConfiguration.get(QUEUE_NAME);
TbPair<Boolean, JsonNode> upgradeResult = tbVersionedNode.upgrade(configurationVersion, oldConfiguration); TbPair<Boolean, JsonNode> upgradeResult = tbVersionedNode.upgrade(configurationVersion, oldConfiguration);
if (upgradeResult.getFirst()) { if (upgradeResult.getFirst()) {
node.setConfiguration(upgradeResult.getSecond()); node.setConfiguration(upgradeResult.getSecond());
if (nodeInfo.getAnnotation().hasQueueName() && queueName != null && queueName.isTextual()) {
node.setQueueName(queueName.asText());
}
} }
} catch (Exception e) { } catch (Exception e) {
try { try {

14
application/src/main/resources/thingsboard.yml

@ -785,6 +785,10 @@ state:
# If 'persistToTelemetry' is changed from 'false' to 'true': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_ts_view;' # If 'persistToTelemetry' is changed from 'false' to 'true': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_ts_view;'
# If 'persistToTelemetry' is changed from 'true' to 'false': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view;' # If 'persistToTelemetry' is changed from 'true' to 'false': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view;'
persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}"
# Millisecond value defining time-to-live for device state telemetry data (e.g. 'active', 'lastActivityTime').
# Used only when state.persistToTelemetry is set to 'true' and Cassandra is used for timeseries data.
# 0 means time-to-live mechanism is disabled.
telemetryTtl: "${STATE_TELEMETRY_TTL:0}"
# Tbel parameters # Tbel parameters
tbel: tbel:
@ -866,6 +870,14 @@ transport:
inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}"
# Interval of periodic check for expired sessions and report of the changes to session last activity time # Interval of periodic check for expired sessions and report of the changes to session last activity time
report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:3000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:3000}"
activity:
# This property specifies the strategy for reporting activity events within each reporting period.
# The accepted values are 'FIRST', 'LAST', 'FIRST_AND_LAST' and 'ALL'.
# - 'FIRST': Only the first activity event in each reporting period is reported.
# - 'LAST': Only the last activity event in the reporting period is reported.
# - 'FIRST_AND_LAST': Both the first and last activity events in the reporting period are reported.
# - 'ALL': All activity events in the reporting period are reported.
reporting_strategy: "${TB_TRANSPORT_ACTIVITY_REPORTING_STRATEGY:LAST}"
json: json:
# Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON
type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}"
@ -1695,4 +1707,4 @@ management:
web: web:
exposure: exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}' include: '${METRICS_ENDPOINTS_EXPOSE:info}'

61
application/src/test/java/org/thingsboard/server/controller/TelemetryControllerTest.java

@ -15,12 +15,17 @@
*/ */
package org.thingsboard.server.controller; package org.thingsboard.server.controller;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.TestPropertySource;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
@ -28,6 +33,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.List; import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.common.data.query.EntityKeyType.TIME_SERIES; import static org.thingsboard.server.common.data.query.EntityKeyType.TIME_SERIES;
@ -51,6 +57,61 @@ public class TelemetryControllerTest extends AbstractControllerTest {
doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", invalidRequestBody, String.class, status().isBadRequest()); doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", invalidRequestBody, String.class, status().isBadRequest());
} }
@Test
public void testTelemetryRequests() throws Exception {
loginTenantAdmin();
Device device = createDevice();
var startTs = 1704899727000L; // Wednesday, January 10 15:15:27 GMT
var endOfWeek1Ts = 1705269600000L; // Monday, January 15, 2024 0:00:00 GMT+02:00
var endOfWeek2Ts = 1705874400000L; // Monday, January 22, 2024 0:00:00 GMT+02:00
var endTs = endOfWeek2Ts + TimeUnit.DAYS.toMillis(1) + TimeUnit.HOURS.toMillis(1); // Monday, January 23, 2024 1:00:00 GMT+02:00
var firstIntervalTs = startTs + (endOfWeek1Ts - startTs) / 2;
var secondIntervalTs = endOfWeek1Ts + (endOfWeek2Ts - endOfWeek1Ts) / 2;
var thirdIntervalTs = endOfWeek2Ts + (endTs - endOfWeek2Ts) / 2;
var middleOfTheInterval = startTs + (endTs - startTs) / 2;
tsService.save(tenantId, device.getId(), new BasicTsKvEntry(1704899728000L, new LongDataEntry("t", 1L))); // Wednesday, January 10 15:15:28 GMT
tsService.save(tenantId, device.getId(), new BasicTsKvEntry(1704899729000L, new LongDataEntry("t", 3L))); // Wednesday, January 10 15:15:29 GMT
tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek1Ts, new LongDataEntry("t", 2L))); // Monday, January 15, 2024 0:00:00 GMT+02:00
tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek1Ts + 1000, new LongDataEntry("t", 5L))); // Monday, January 15, 2024 0:00:01 GMT+02:00
tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek2Ts, new LongDataEntry("t", 9L))); // Monday, January 22, 2024 0:00:00 GMT+02:00
tsService.save(tenantId, device.getId(), new BasicTsKvEntry(endOfWeek2Ts + 1000, new LongDataEntry("t", 2L))); // Monday, January 22, 2024 0:00:01 GMT+02:00
ObjectNode result = doGetAsync("/api/plugins/telemetry/DEVICE/" + device.getId() +
"/values/timeseries?keys=t&startTs={startTs}&endTs={endTs}&agg={agg}&intervalType={intervalType}&timeZone={timeZone}",
ObjectNode.class, startTs, endTs, "SUM", "WEEK_ISO", "Europe/Kyiv");
Assert.assertNotNull(result);
Assert.assertNotNull(result.get("t"));
Assert.assertEquals(3, result.get("t").size());
var firstIntervalResult = result.get("t").get(0);
Assert.assertEquals(4L, firstIntervalResult.get("value").asLong());
Assert.assertEquals(firstIntervalTs, firstIntervalResult.get("ts").asLong());
var secondIntervalResult = result.get("t").get(1);
Assert.assertEquals(7L, secondIntervalResult.get("value").asLong());
Assert.assertEquals(secondIntervalTs, secondIntervalResult.get("ts").asLong());
var thirdIntervalResult = result.get("t").get(2);
Assert.assertEquals(11L, thirdIntervalResult.get("value").asLong());
Assert.assertEquals(thirdIntervalTs, thirdIntervalResult.get("ts").asLong());
result = doGetAsync("/api/plugins/telemetry/DEVICE/" + device.getId() +
"/values/timeseries?keys=t&startTs={startTs}&endTs={endTs}&agg={agg}&intervalType={intervalType}&timeZone={timeZone}",
ObjectNode.class, startTs, endTs, "SUM", "MONTH", "Europe/Kyiv");
Assert.assertNotNull(result);
Assert.assertNotNull(result.get("t"));
Assert.assertEquals(1, result.get("t").size());
var monthResult = result.get("t").get(0);
Assert.assertEquals(22L, monthResult.get("value").asLong());
Assert.assertEquals(middleOfTheInterval, monthResult.get("ts").asLong());
}
@Test @Test
public void testDeleteAllTelemetryWithLatest() throws Exception { public void testDeleteAllTelemetryWithLatest() throws Exception {
loginTenantAdmin(); loginTenantAdmin();

75
application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java

@ -25,8 +25,10 @@ import io.netty.handler.codec.mqtt.MqttQoS;
import org.awaitility.Awaitility; import org.awaitility.Awaitility;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.TestPropertySource;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
@ -44,15 +46,17 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg; import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg;
import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg;
@ -75,6 +79,7 @@ import java.util.Optional;
import java.util.Random; import java.util.Random;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@ -86,6 +91,9 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
private static final String DEFAULT_DEVICE_TYPE = "default"; private static final String DEFAULT_DEVICE_TYPE = "default";
@Autowired
protected EdgeService edgeService;
@Test @Test
public void testDevices() throws Exception { public void testDevices() throws Exception {
// create device and assign to edge; update device // create device and assign to edge; update device
@ -769,6 +777,54 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
} }
} }
@Test
public void testVerifyProcessCorrectEdgeUpdateToDeviceActorOnUnassignFromDifferentEdge() throws Exception {
Device device = saveDeviceOnCloudAndVerifyDeliveryToEdge();
// assign device to another edge
Edge tmpEdge = doPost("/api/edge", constructEdge("Test Tmp Edge", "test"), Edge.class);
doPost("/api/edge/" + tmpEdge.getUuidId()
+ "/device/" + device.getUuidId(), Device.class);
List<EdgeId> relatedEdgeIds = edgeService.findAllRelatedEdgeIds(tenantId, device.getId());
Assert.assertEquals(2, relatedEdgeIds.size());
// unassign device from edge
doDelete("/api/edge/" + edge.getUuidId()
+ "/device/" + device.getUuidId(), Device.class);
relatedEdgeIds = edgeService.findAllRelatedEdgeIds(tenantId, device.getId());
Assert.assertEquals(1, relatedEdgeIds.size());
Assert.assertEquals(tmpEdge.getId(), relatedEdgeIds.get(0));
// clean up stored edge events
edgeEventService.cleanupEvents(1);
// perform rpc call to verify edgeId in DeviceActorMessageProcessor updated properly
doPostAsync(
"/api/rpc/oneway/" + device.getId().getId().toString(),
JacksonUtil.toString(createDefaultRpc()),
String.class,
status().isOk());
final AtomicReference<PageData<EdgeEvent>> resultRef = new AtomicReference<>();
Awaitility.await()
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> {
PageData<EdgeEvent> result = edgeEventService.findEdgeEvents(tenantId, tmpEdge.getId(), 0L, null, new TimePageLink(1));
resultRef.set(result);
return result != null && result.getData().size() == 1;
});
PageData<EdgeEvent> result = resultRef.get();
EdgeEvent edgeEvent = result.getData().get(0);
Assert.assertEquals(EdgeEventActionType.RPC_CALL, edgeEvent.getAction());
Assert.assertEquals(EdgeEventType.DEVICE, edgeEvent.getType());
Assert.assertEquals(tmpEdge.getId(), edgeEvent.getEdgeId());
Assert.assertEquals(device.getId().getId(), edgeEvent.getEntityId());
// clean up tmp edge
doDelete("/api/edge/" + tmpEdge.getId().getId().toString()).andExpect(status().isOk());
}
private Device buildDeviceForUplinkMsg(String name, String type) { private Device buildDeviceForUplinkMsg(String name, String type) {
Device device = new Device(); Device device = new Device();
device.setId(new DeviceId(UUID.randomUUID())); device.setId(new DeviceId(UUID.randomUUID()));
@ -778,7 +834,6 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
return device; return device;
} }
private DeviceCredentials buildDeviceCredentialsForUplinkMsg(DeviceId deviceId) { private DeviceCredentials buildDeviceCredentialsForUplinkMsg(DeviceId deviceId) {
DeviceCredentials deviceCredentials = new DeviceCredentials(); DeviceCredentials deviceCredentials = new DeviceCredentials();
deviceCredentials.setDeviceId(deviceId); deviceCredentials.setDeviceId(deviceId);
@ -786,4 +841,20 @@ public class DeviceEdgeTest extends AbstractEdgeTest {
deviceCredentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN); deviceCredentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
return deviceCredentials; return deviceCredentials;
} }
private ObjectNode createDefaultRpc() {
ObjectNode rpc = JacksonUtil.newObjectNode();
rpc.put("method", "setGpio");
ObjectNode params = JacksonUtil.newObjectNode();
params.put("pin", 7);
params.put("value", 1);
rpc.set("params", params);
rpc.put("persistent", true);
rpc.put("timeout", 5000);
return rpc;
}
} }

5
application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java

@ -29,11 +29,12 @@ import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService;
@ -73,6 +74,8 @@ public class DefaultTbClusterServiceTest {
@MockBean @MockBean
protected GatewayNotificationsService gatewayNotificationsService; protected GatewayNotificationsService gatewayNotificationsService;
@MockBean @MockBean
protected EdgeService edgeService;
@MockBean
protected PartitionService partitionService; protected PartitionService partitionService;
@MockBean @MockBean
protected TbQueueProducerProvider producerProvider; protected TbQueueProducerProvider producerProvider;

1
common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java

@ -136,5 +136,6 @@ public class DataConstants {
public static final String HP_QUEUE_TOPIC = "tb_rule_engine.hp"; public static final String HP_QUEUE_TOPIC = "tb_rule_engine.hp";
public static final String SQ_QUEUE_NAME = "SequentialByOriginator"; public static final String SQ_QUEUE_NAME = "SequentialByOriginator";
public static final String SQ_QUEUE_TOPIC = "tb_rule_engine.sq"; public static final String SQ_QUEUE_TOPIC = "tb_rule_engine.sq";
public static final String QUEUE_NAME = "queueName";
} }

92
common/data/src/main/java/org/thingsboard/server/common/data/kv/AggregationParams.java

@ -0,0 +1,92 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.kv;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.StringUtils;
import java.time.DateTimeException;
import java.time.ZoneId;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@AllArgsConstructor
@EqualsAndHashCode
@Slf4j
public class AggregationParams {
private static final Map<String, String> TZ_LINKS = Map.of("EST", "America/New_York", "GMT+0", "GMT", "GMT-0", "GMT", "HST", "US/Hawaii", "MST", "America/Phoenix", "ROC", "Asia/Taipei");
@Getter
private final Aggregation aggregation;
@Getter
private final IntervalType intervalType;
@Getter
private final ZoneId tzId;
private final long interval;
public static AggregationParams none() {
return new AggregationParams(Aggregation.NONE, null, null, 0L);
}
public static AggregationParams milliseconds(Aggregation aggregationType, long aggregationIntervalMs) {
return new AggregationParams(aggregationType, IntervalType.MILLISECONDS, null, aggregationIntervalMs);
}
public static AggregationParams calendar(Aggregation aggregationType, IntervalType intervalType, String tzIdStr) {
return calendar(aggregationType, intervalType, getZoneId(tzIdStr));
}
public static AggregationParams calendar(Aggregation aggregationType, IntervalType intervalType, ZoneId tzId) {
return new AggregationParams(aggregationType, intervalType, tzId, 0L);
}
public static AggregationParams of(Aggregation aggregation, IntervalType intervalType, ZoneId tzId, long interval) {
return new AggregationParams(aggregation, intervalType, tzId, interval);
}
public long getInterval() {
if (intervalType == null) {
return 0L;
} else {
switch (intervalType) {
case WEEK:
case WEEK_ISO:
return TimeUnit.DAYS.toMillis(7);
case MONTH:
return TimeUnit.DAYS.toMillis(30);
case QUARTER:
return TimeUnit.DAYS.toMillis(90);
default:
return interval;
}
}
}
private static ZoneId getZoneId(String tzIdStr) {
if (StringUtils.isEmpty(tzIdStr)) {
return ZoneId.systemDefault();
}
try {
return ZoneId.of(tzIdStr, TZ_LINKS);
} catch (DateTimeException e) {
log.warn("[{}] Failed to convert the time zone. Fallback to default.", tzIdStr);
return ZoneId.systemDefault();
}
}
}

26
common/data/src/main/java/org/thingsboard/server/common/data/kv/BaseReadTsKvQuery.java

@ -18,41 +18,47 @@ package org.thingsboard.server.common.data.kv;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import java.time.ZoneId;
@Data @Data
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
public class BaseReadTsKvQuery extends BaseTsKvQuery implements ReadTsKvQuery { public class BaseReadTsKvQuery extends BaseTsKvQuery implements ReadTsKvQuery {
private final long interval; private final AggregationParams aggParameters;
private final int limit; private final int limit;
private final Aggregation aggregation;
private final String order; private final String order;
public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation) { public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation) {
this(key, startTs, endTs, interval, limit, aggregation, "DESC"); this(key, startTs, endTs, interval, limit, aggregation, "DESC");
} }
public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, String order) { public BaseReadTsKvQuery(String key, long startTs, long endTs, long interval, int limit, Aggregation aggregation, String descOrder) {
this(key, startTs, endTs, AggregationParams.of(aggregation, IntervalType.MILLISECONDS, ZoneId.systemDefault(), interval), limit, descOrder);
}
public BaseReadTsKvQuery(String key, long startTs, long endTs, AggregationParams parameters, int limit) {
this(key, startTs, endTs, parameters, limit, "DESC");
}
public BaseReadTsKvQuery(String key, long startTs, long endTs, AggregationParams parameters, int limit, String order) {
super(key, startTs, endTs); super(key, startTs, endTs);
this.interval = interval; this.aggParameters = parameters;
this.limit = limit; this.limit = limit;
this.aggregation = aggregation;
this.order = order; this.order = order;
} }
public BaseReadTsKvQuery(String key, long startTs, long endTs) { public BaseReadTsKvQuery(String key, long startTs, long endTs) {
this(key, startTs, endTs, endTs - startTs, 1, Aggregation.AVG, "DESC"); this(key, startTs, endTs, AggregationParams.milliseconds(Aggregation.AVG, endTs - startTs), 1, "DESC");
} }
public BaseReadTsKvQuery(String key, long startTs, long endTs, int limit, String order) { public BaseReadTsKvQuery(String key, long startTs, long endTs, int limit, String order) {
this(key, startTs, endTs, endTs - startTs, limit, Aggregation.NONE, order); this(key, startTs, endTs, AggregationParams.none(), limit, order);
} }
public BaseReadTsKvQuery(ReadTsKvQuery query, long startTs, long endTs) { public BaseReadTsKvQuery(ReadTsKvQuery query, long startTs, long endTs) {
super(query.getId(), query.getKey(), startTs, endTs); super(query.getId(), query.getKey(), startTs, endTs);
this.interval = query.getInterval(); this.aggParameters = query.getAggParameters();
this.limit = query.getLimit(); this.limit = query.getLimit();
this.aggregation = query.getAggregation();
this.order = query.getOrder(); this.order = query.getOrder();
} }
} }

22
common/data/src/main/java/org/thingsboard/server/common/data/kv/IntervalType.java

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.kv;
public enum IntervalType {
MILLISECONDS, WEEK/*Sunday-Saturday*/, WEEK_ISO/*Monday-Sunday*/, MONTH, QUARTER
}

12
common/data/src/main/java/org/thingsboard/server/common/data/kv/ReadTsKvQuery.java

@ -17,11 +17,17 @@ package org.thingsboard.server.common.data.kv;
public interface ReadTsKvQuery extends TsKvQuery { public interface ReadTsKvQuery extends TsKvQuery {
long getInterval(); AggregationParams getAggParameters();
int getLimit(); default long getInterval(){
return getAggParameters().getInterval();
}
default Aggregation getAggregation() {
return getAggParameters().getAggregation();
}
Aggregation getAggregation(); int getLimit();
String getOrder(); String getOrder();

7
common/data/src/main/java/org/thingsboard/server/common/data/plugin/ComponentDescriptor.java

@ -55,6 +55,8 @@ public class ComponentDescriptor extends BaseData<ComponentDescriptorId> {
@Length(fieldName = "actions") @Length(fieldName = "actions")
@ApiModelProperty(position = 10, value = "Rule Node Actions. Deprecated. Always null.", accessMode = ApiModelProperty.AccessMode.READ_ONLY) @ApiModelProperty(position = 10, value = "Rule Node Actions. Deprecated. Always null.", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
@Getter @Setter private String actions; @Getter @Setter private String actions;
@ApiModelProperty(position = 11, value = "Indicates that the RuleNode supports queue name configuration.", accessMode = ApiModelProperty.AccessMode.READ_ONLY, example = "true")
@Getter @Setter private boolean hasQueueName;
public ComponentDescriptor() { public ComponentDescriptor() {
super(); super();
@ -74,6 +76,7 @@ public class ComponentDescriptor extends BaseData<ComponentDescriptorId> {
this.configurationDescriptor = plugin.getConfigurationDescriptor(); this.configurationDescriptor = plugin.getConfigurationDescriptor();
this.configurationVersion = plugin.getConfigurationVersion(); this.configurationVersion = plugin.getConfigurationVersion();
this.actions = plugin.getActions(); this.actions = plugin.getActions();
this.hasQueueName = plugin.isHasQueueName();
} }
@ApiModelProperty(position = 1, value = "JSON object with the descriptor Id. " + @ApiModelProperty(position = 1, value = "JSON object with the descriptor Id. " +
@ -104,6 +107,8 @@ public class ComponentDescriptor extends BaseData<ComponentDescriptorId> {
if (!Objects.equals(actions, that.actions)) return false; if (!Objects.equals(actions, that.actions)) return false;
if (!Objects.equals(configurationDescriptor, that.configurationDescriptor)) return false; if (!Objects.equals(configurationDescriptor, that.configurationDescriptor)) return false;
if (configurationVersion != that.configurationVersion) return false; if (configurationVersion != that.configurationVersion) return false;
if (clusteringMode != that.clusteringMode) return false;
if (hasQueueName != that.isHasQueueName()) return false;
return Objects.equals(clazz, that.clazz); return Objects.equals(clazz, that.clazz);
} }
@ -115,6 +120,8 @@ public class ComponentDescriptor extends BaseData<ComponentDescriptorId> {
result = 31 * result + (name != null ? name.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0);
result = 31 * result + (clazz != null ? clazz.hashCode() : 0); result = 31 * result + (clazz != null ? clazz.hashCode() : 0);
result = 31 * result + (actions != null ? actions.hashCode() : 0); result = 31 * result + (actions != null ? actions.hashCode() : 0);
result = 31 * result + (clusteringMode != null ? clusteringMode.hashCode() : 0);
result = 31 * result + (hasQueueName ? 1 : 0);
return result; return result;
} }

6
common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java

@ -50,9 +50,11 @@ public class RuleNode extends BaseDataWithAdditionalInfo<RuleNodeId> implements
private boolean debugMode; private boolean debugMode;
@ApiModelProperty(position = 7, value = "Enable/disable singleton mode. ", example = "false") @ApiModelProperty(position = 7, value = "Enable/disable singleton mode. ", example = "false")
private boolean singletonMode; private boolean singletonMode;
@ApiModelProperty(position = 8, value = "Version of rule node configuration. ", example = "0") @ApiModelProperty(position = 8, value = "Queue name. ", example = "Main")
private String queueName;
@ApiModelProperty(position = 9, value = "Version of rule node configuration. ", example = "0")
private int configurationVersion; private int configurationVersion;
@ApiModelProperty(position = 9, value = "JSON with the rule node configuration. Structure depends on the rule node implementation.", dataType = "com.fasterxml.jackson.databind.JsonNode") @ApiModelProperty(position = 10, value = "JSON with the rule node configuration. Structure depends on the rule node implementation.", dataType = "com.fasterxml.jackson.databind.JsonNode")
private transient JsonNode configuration; private transient JsonNode configuration;
@JsonIgnore @JsonIgnore
private byte[] configurationBytes; private byte[] configurationBytes;

2
common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/DefaultCoapClientContext.java

@ -212,7 +212,7 @@ public class DefaultCoapClientContext implements CoapClientContext {
public void reportActivity() { public void reportActivity() {
for (TbCoapClientState state : clients.values()) { for (TbCoapClientState state : clients.values()) {
if (state.getSession() != null) { if (state.getSession() != null) {
transportService.reportActivity(state.getSession()); transportService.recordActivity(state.getSession());
} }
} }
} }

10
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java

@ -322,7 +322,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
case PINGREQ: case PINGREQ:
if (checkConnected(ctx, msg)) { if (checkConnected(ctx, msg)) {
ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0)));
transportService.reportActivity(deviceSessionCtx.getSessionInfo()); transportService.recordActivity(deviceSessionCtx.getSessionInfo());
} }
break; break;
case DISCONNECT: case DISCONNECT:
@ -350,7 +350,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
if (topicName.startsWith(MqttTopics.BASE_GATEWAY_API_TOPIC)) { if (topicName.startsWith(MqttTopics.BASE_GATEWAY_API_TOPIC)) {
if (gatewaySessionHandler != null) { if (gatewaySessionHandler != null) {
handleGatewayPublishMsg(ctx, topicName, msgId, mqttMsg); handleGatewayPublishMsg(ctx, topicName, msgId, mqttMsg);
transportService.reportActivity(deviceSessionCtx.getSessionInfo()); transportService.recordActivity(deviceSessionCtx.getSessionInfo());
} else { } else {
log.error("[gatewaySessionHandler] is null, [{}] Failed to process publish msg [{}][{}]", sessionId, topicName, msgId); log.error("[gatewaySessionHandler] is null, [{}] Failed to process publish msg [{}][{}]", sessionId, topicName, msgId);
} }
@ -526,7 +526,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
transportService.process(deviceSessionCtx.getSessionInfo(), getAttributeMsg, getPubAckCallback(ctx, msgId, getAttributeMsg)); transportService.process(deviceSessionCtx.getSessionInfo(), getAttributeMsg, getPubAckCallback(ctx, msgId, getAttributeMsg));
attrReqTopicType = TopicType.V2; attrReqTopicType = TopicType.V2;
} else { } else {
transportService.reportActivity(deviceSessionCtx.getSessionInfo()); transportService.recordActivity(deviceSessionCtx.getSessionInfo());
ack(ctx, msgId, ReturnCode.TOPIC_NAME_INVALID); ack(ctx, msgId, ReturnCode.TOPIC_NAME_INVALID);
} }
} catch (AdaptorException e) { } catch (AdaptorException e) {
@ -796,7 +796,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
} }
} }
if (!activityReported) { if (!activityReported) {
transportService.reportActivity(deviceSessionCtx.getSessionInfo()); transportService.recordActivity(deviceSessionCtx.getSessionInfo());
} }
ctx.writeAndFlush(createSubAckMessage(mqttMsg.variableHeader().messageId(), grantedQoSList)); ctx.writeAndFlush(createSubAckMessage(mqttMsg.variableHeader().messageId(), grantedQoSList));
} }
@ -894,7 +894,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
} }
} }
if (!activityReported) { if (!activityReported) {
transportService.reportActivity(deviceSessionCtx.getSessionInfo()); transportService.recordActivity(deviceSessionCtx.getSessionInfo());
} }
ctx.writeAndFlush(createUnSubAckMessage(mqttMsg.variableHeader().messageId(), unSubResults)); ctx.writeAndFlush(createUnSubAckMessage(mqttMsg.variableHeader().messageId(), unSubResults));
} }

2
common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java

@ -416,7 +416,7 @@ public class SnmpTransportService implements TbTransportService, CommandResponde
} }
private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) {
transportService.reportActivity(sessionInfo); transportService.recordActivity(sessionInfo);
} }

2
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java

@ -145,7 +145,7 @@ public interface TransportService {
SessionMetaData registerSyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout); SessionMetaData registerSyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout);
void reportActivity(SessionInfoProto sessionInfo); void recordActivity(SessionInfoProto sessionInfo);
void lifecycleEvent(TenantId tenantId, DeviceId deviceId, ComponentLifecycleEvent eventType, boolean success, Throwable error); void lifecycleEvent(TenantId tenantId, DeviceId deviceId, ComponentLifecycleEvent eventType, boolean success, Throwable error);

187
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/AbstractActivityManager.java

@ -0,0 +1,187 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.transport.activity.strategy.ActivityStrategy;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
public abstract class AbstractActivityManager<Key, Metadata> implements ActivityManager<Key> {
private final ConcurrentMap<Key, ActivityStateWrapper> states = new ConcurrentHashMap<>();
@Autowired
protected SchedulerComponent scheduler;
@Data
private class ActivityStateWrapper {
private volatile ActivityState<Metadata> state;
private volatile long lastReportedTime;
private volatile ActivityStrategy strategy;
}
protected void init() {
var reportingPeriodMillis = getReportingPeriodMillis();
scheduler.scheduleAtFixedRate(this::onReportingPeriodEnd, new Random().nextInt((int) reportingPeriodMillis), reportingPeriodMillis, TimeUnit.MILLISECONDS);
}
protected abstract long getReportingPeriodMillis();
protected abstract ActivityState<Metadata> createNewState(Key key);
protected abstract ActivityStrategy getStrategy();
protected abstract ActivityState<Metadata> updateState(Key key, ActivityState<Metadata> state);
protected abstract boolean hasExpired(long lastRecordedTime);
protected abstract void onStateExpiry(Key key, Metadata metadata);
protected abstract void reportActivity(Key key, Metadata metadata, long timeToReport, ActivityReportCallback<Key> callback);
@Override
public void onActivity(Key key, long newLastRecordedTime) {
if (key == null) {
log.error("Failed to process activity event: provided activity key is null.");
return;
}
log.debug("Received activity event for key: [{}]", key);
var shouldReport = new AtomicBoolean(false);
var lastRecordedTime = new AtomicLong();
var lastReportedTime = new AtomicLong();
var metadata = new AtomicReference<Metadata>();
var activityStateWrapper = states.compute(key, (__, stateWrapper) -> {
if (stateWrapper == null) {
var newState = createNewState(key);
if (newState == null) {
return null;
}
stateWrapper = new ActivityStateWrapper();
stateWrapper.setState(newState);
stateWrapper.setStrategy(getStrategy());
}
var state = stateWrapper.getState();
if (state.getLastRecordedTime() < newLastRecordedTime) {
state.setLastRecordedTime(newLastRecordedTime);
}
shouldReport.set(stateWrapper.getStrategy().onActivity());
lastRecordedTime.set(state.getLastRecordedTime());
lastReportedTime.set(stateWrapper.getLastReportedTime());
metadata.set(state.getMetadata());
return stateWrapper;
});
if (activityStateWrapper == null) {
return;
}
if (shouldReport.get() && lastReportedTime.get() < lastRecordedTime.get()) {
log.debug("Going to report first activity event for key: [{}].", key);
reportActivity(key, metadata.get(), lastRecordedTime.get(), new ActivityReportCallback<>() {
@Override
public void onSuccess(Key key, long reportedTime) {
updateLastReportedTime(key, reportedTime);
}
@Override
public void onFailure(Key key, Throwable t) {
log.debug("Failed to report first activity event for key: [{}].", key, t);
}
});
}
}
@Override
public void onReportingPeriodEnd() {
log.debug("Going to end reporting period.");
for (Map.Entry<Key, ActivityStateWrapper> entry : states.entrySet()) {
var key = entry.getKey();
var stateWrapper = entry.getValue();
var currentState = stateWrapper.getState();
long lastRecordedTime = currentState.getLastRecordedTime();
long lastReportedTime = stateWrapper.getLastReportedTime();
var metadata = currentState.getMetadata();
boolean hasExpired;
boolean shouldReport;
var updatedState = updateState(key, currentState);
if (updatedState != null) {
stateWrapper.setState(updatedState);
lastRecordedTime = updatedState.getLastRecordedTime();
metadata = updatedState.getMetadata();
hasExpired = hasExpired(lastRecordedTime);
shouldReport = stateWrapper.getStrategy().onReportingPeriodEnd();
} else {
states.remove(key);
hasExpired = false;
shouldReport = true;
}
if (hasExpired) {
states.remove(key);
onStateExpiry(key, metadata);
shouldReport = true;
}
if (shouldReport && lastReportedTime < lastRecordedTime) {
log.debug("Going to report last activity event for key: [{}].", key);
reportActivity(key, metadata, lastRecordedTime, new ActivityReportCallback<>() {
@Override
public void onSuccess(Key key, long reportedTime) {
updateLastReportedTime(key, reportedTime);
}
@Override
public void onFailure(Key key, Throwable t) {
log.debug("Failed to report last activity event for key: [{}].", key, t);
}
});
}
}
}
@Override
public long getLastRecordedTime(Key key) {
ActivityStateWrapper stateWrapper = states.get(key);
return stateWrapper == null ? 0L : stateWrapper.getState().getLastRecordedTime();
}
private void updateLastReportedTime(Key key, long newLastReportedTime) {
states.computeIfPresent(key, (__, stateWrapper) -> {
stateWrapper.setLastReportedTime(Math.max(stateWrapper.getLastReportedTime(), newLastReportedTime));
return stateWrapper;
});
}
}

26
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/ActivityManager.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity;
public interface ActivityManager<Key> {
void onActivity(Key key, long activityTimeMillis);
void onReportingPeriodEnd();
long getLastRecordedTime(Key key);
}

24
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/ActivityReportCallback.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity;
public interface ActivityReportCallback<Key> {
void onSuccess(Key key, long reportedTime);
void onFailure(Key key, Throwable t);
}

12
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeConfiguration.java → common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/ActivityState.java

@ -13,18 +13,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.rule.engine.flow; package org.thingsboard.server.common.transport.activity;
import lombok.Data; import lombok.Data;
import org.thingsboard.rule.engine.api.NodeConfiguration;
@Data @Data
public class TbCheckpointNodeConfiguration implements NodeConfiguration<TbCheckpointNodeConfiguration> { public class ActivityState<Metadata> {
private String queueName; private volatile long lastRecordedTime;
private volatile Metadata metadata;
@Override
public TbCheckpointNodeConfiguration defaultConfiguration() {
return new TbCheckpointNodeConfiguration();
}
} }

24
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/ActivityStrategy.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
public interface ActivityStrategy {
boolean onActivity();
boolean onReportingPeriodEnd();
}

47
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/ActivityStrategyType.java

@ -0,0 +1,47 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
public enum ActivityStrategyType {
ALL {
@Override
public ActivityStrategy toStrategy() {
return AllEventsActivityStrategy.getInstance();
}
},
FIRST {
@Override
public ActivityStrategy toStrategy() {
return new FirstEventActivityStrategy();
}
},
LAST {
@Override
public ActivityStrategy toStrategy() {
return LastEventActivityStrategy.getInstance();
}
},
FIRST_AND_LAST {
@Override
public ActivityStrategy toStrategy() {
return new FirstAndLastEventActivityStrategy();
}
};
public abstract ActivityStrategy toStrategy();
}

30
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/SessionActivityData.java → common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/AllEventsActivityStrategy.java

@ -13,27 +13,27 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.common.transport.service; package org.thingsboard.server.common.transport.activity.strategy;
import lombok.Data; public final class AllEventsActivityStrategy implements ActivityStrategy {
import org.thingsboard.server.gen.transport.TransportProtos;
/** private static final AllEventsActivityStrategy INSTANCE = new AllEventsActivityStrategy();
* Created by ashvayka on 15.10.18.
*/
@Data
public class SessionActivityData {
private volatile TransportProtos.SessionInfoProto sessionInfo; private AllEventsActivityStrategy() {
private volatile long lastActivityTime; }
private volatile long lastReportedActivityTime;
public static AllEventsActivityStrategy getInstance() {
return INSTANCE;
}
SessionActivityData(TransportProtos.SessionInfoProto sessionInfo) { @Override
this.sessionInfo = sessionInfo; public boolean onActivity() {
return true;
} }
void updateLastActivityTime() { @Override
this.lastActivityTime = System.currentTimeMillis(); public boolean onReportingPeriodEnd() {
return true;
} }
} }

40
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/FirstAndLastEventActivityStrategy.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
import lombok.EqualsAndHashCode;
@EqualsAndHashCode
public final class FirstAndLastEventActivityStrategy implements ActivityStrategy {
private boolean firstEventReceived;
@Override
public synchronized boolean onActivity() {
if (!firstEventReceived) {
firstEventReceived = true;
return true;
}
return false;
}
@Override
public synchronized boolean onReportingPeriodEnd() {
firstEventReceived = false;
return true;
}
}

40
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/FirstEventActivityStrategy.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
import lombok.EqualsAndHashCode;
@EqualsAndHashCode
public final class FirstEventActivityStrategy implements ActivityStrategy {
private boolean firstEventReceived;
@Override
public synchronized boolean onActivity() {
if (!firstEventReceived) {
firstEventReceived = true;
return true;
}
return false;
}
@Override
public synchronized boolean onReportingPeriodEnd() {
firstEventReceived = false;
return false;
}
}

39
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/activity/strategy/LastEventActivityStrategy.java

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
public final class LastEventActivityStrategy implements ActivityStrategy {
private static final LastEventActivityStrategy INSTANCE = new LastEventActivityStrategy();
private LastEventActivityStrategy() {
}
public static LastEventActivityStrategy getInstance() {
return INSTANCE;
}
@Override
public boolean onActivity() {
return false;
}
@Override
public boolean onReportingPeriodEnd() {
return true;
}
}

124
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java

@ -50,9 +50,9 @@ import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.limit.LimitedApi; import org.thingsboard.server.common.data.limit.LimitedApi;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.notification.rule.trigger.RateLimitsTrigger; import org.thingsboard.server.common.data.notification.rule.trigger.RateLimitsTrigger;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.common.data.rpc.RpcStatus;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
@ -96,9 +96,9 @@ import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.AsyncCallbackTemplate; import org.thingsboard.server.queue.common.AsyncCallbackTemplate;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TopicService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.provider.TbTransportQueueFactory; import org.thingsboard.server.queue.provider.TbTransportQueueFactory;
import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.scheduler.SchedulerComponent;
@ -109,16 +109,13 @@ import org.thingsboard.server.queue.util.TbTransportComponent;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Random; import java.util.Random;
import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
@ -133,14 +130,12 @@ import java.util.stream.Collectors;
@Slf4j @Slf4j
@Service @Service
@TbTransportComponent @TbTransportComponent
public class DefaultTransportService implements TransportService { public class DefaultTransportService extends TransportActivityManager implements TransportService {
public static final String OVERWRITE_ACTIVITY_TIME = "overwriteActivityTime"; public static final String OVERWRITE_ACTIVITY_TIME = "overwriteActivityTime";
public static final String SESSION_EXPIRED_MESSAGE = "Session has expired due to last activity time!"; public static final TransportProtos.SessionEventMsg SESSION_EVENT_MSG_OPEN = TransportProtos.SessionEventMsg.newBuilder()
public static final TransportProtos.SessionEventMsg SESSION_EVENT_MSG_OPEN = getSessionEventMsg(TransportProtos.SessionEvent.OPEN); .setSessionType(TransportProtos.SessionType.ASYNC)
public static final TransportProtos.SessionEventMsg SESSION_EVENT_MSG_CLOSED = getSessionEventMsg(TransportProtos.SessionEvent.CLOSED); .setEvent(TransportProtos.SessionEvent.OPEN).build();
public static final TransportProtos.SessionCloseNotificationProto SESSION_CLOSE_NOTIFICATION_PROTO = TransportProtos.SessionCloseNotificationProto.newBuilder()
.setMessage(SESSION_EXPIRED_MESSAGE).build();
public static final TransportProtos.SubscribeToAttributeUpdatesMsg SUBSCRIBE_TO_ATTRIBUTE_UPDATES_ASYNC_MSG = TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder() public static final TransportProtos.SubscribeToAttributeUpdatesMsg SUBSCRIBE_TO_ATTRIBUTE_UPDATES_ASYNC_MSG = TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder()
.setSessionType(TransportProtos.SessionType.ASYNC).build(); .setSessionType(TransportProtos.SessionType.ASYNC).build();
public static final TransportProtos.SubscribeToRPCMsg SUBSCRIBE_TO_RPC_ASYNC_MSG = TransportProtos.SubscribeToRPCMsg.newBuilder() public static final TransportProtos.SubscribeToRPCMsg SUBSCRIBE_TO_RPC_ASYNC_MSG = TransportProtos.SubscribeToRPCMsg.newBuilder()
@ -152,10 +147,6 @@ public class DefaultTransportService implements TransportService {
private boolean logEnabled; private boolean logEnabled;
@Value("${transport.log.max_length:1024}") @Value("${transport.log.max_length:1024}")
private int logMaxLength; private int logMaxLength;
@Value("${transport.sessions.inactivity_timeout}")
private long sessionInactivityTimeout;
@Value("${transport.sessions.report_timeout}")
private long sessionReportTimeout;
@Value("${transport.client_side_rpc.timeout:60000}") @Value("${transport.client_side_rpc.timeout:60000}")
private long clientSideRpcTimeout; private long clientSideRpcTimeout;
@Value("${queue.transport.poll_interval}") @Value("${queue.transport.poll_interval}")
@ -182,11 +173,9 @@ public class DefaultTransportService implements TransportService {
private final TransportRateLimitService rateLimitService; private final TransportRateLimitService rateLimitService;
private final DataDecodingEncodingService dataDecodingEncodingService; private final DataDecodingEncodingService dataDecodingEncodingService;
private final SchedulerComponent scheduler;
private final ApplicationEventPublisher eventPublisher; private final ApplicationEventPublisher eventPublisher;
private final TransportResourceCache transportResourceCache; private final TransportResourceCache transportResourceCache;
private final NotificationRuleProcessor notificationRuleProcessor; private final NotificationRuleProcessor notificationRuleProcessor;
private final EntityLimitsCache entityLimitsCache; private final EntityLimitsCache entityLimitsCache;
protected TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> transportApiRequestTemplate; protected TbQueueRequestTemplate<TbProtoQueueMsg<TransportApiRequestMsg>, TbProtoQueueMsg<TransportApiResponseMsg>> transportApiRequestTemplate;
@ -201,8 +190,6 @@ public class DefaultTransportService implements TransportService {
protected ExecutorService transportCallbackExecutor; protected ExecutorService transportCallbackExecutor;
private ExecutorService mainConsumerExecutor; private ExecutorService mainConsumerExecutor;
public final ConcurrentMap<UUID, SessionMetaData> sessions = new ConcurrentHashMap<>();
private final ConcurrentMap<UUID, SessionActivityData> sessionsActivity = new ConcurrentHashMap<>();
private final Map<String, RpcRequestMetadata> toServerRpcPendingMap = new ConcurrentHashMap<>(); private final Map<String, RpcRequestMetadata> toServerRpcPendingMap = new ConcurrentHashMap<>();
private volatile boolean stopped = false; private volatile boolean stopped = false;
@ -238,11 +225,11 @@ public class DefaultTransportService implements TransportService {
@PostConstruct @PostConstruct
public void init() { public void init() {
super.init();
this.ruleEngineProducerStats = statsFactory.createMessagesStats(StatsType.RULE_ENGINE.getName() + ".producer"); this.ruleEngineProducerStats = statsFactory.createMessagesStats(StatsType.RULE_ENGINE.getName() + ".producer");
this.tbCoreProducerStats = statsFactory.createMessagesStats(StatsType.CORE.getName() + ".producer"); this.tbCoreProducerStats = statsFactory.createMessagesStats(StatsType.CORE.getName() + ".producer");
this.transportApiStats = statsFactory.createMessagesStats(StatsType.TRANSPORT.getName() + ".producer"); this.transportApiStats = statsFactory.createMessagesStats(StatsType.TRANSPORT.getName() + ".producer");
this.transportCallbackExecutor = ThingsBoardExecutors.newWorkStealingPool(20, getClass()); this.transportCallbackExecutor = ThingsBoardExecutors.newWorkStealingPool(20, getClass());
this.scheduler.scheduleAtFixedRate(this::checkInactivityAndReportActivity, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS);
this.scheduler.scheduleAtFixedRate(this::invalidateRateLimits, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS); this.scheduler.scheduleAtFixedRate(this::invalidateRateLimits, new Random().nextInt((int) sessionReportTimeout), sessionReportTimeout, TimeUnit.MILLISECONDS);
transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate(); transportApiRequestTemplate = queueProvider.createTransportApiRequestTemplate();
transportApiRequestTemplate.setMessagesStats(transportApiStats); transportApiRequestTemplate.setMessagesStats(transportApiStats);
@ -558,7 +545,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback<Void> callback) { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SessionEventMsg msg, TransportServiceCallback<Void> callback) {
if (checkLimits(sessionInfo, msg, callback)) { if (checkLimits(sessionInfo, msg, callback)) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
.setSessionEvent(msg).build(), callback); .setSessionEvent(msg).build(), callback);
} }
@ -578,7 +565,7 @@ public class DefaultTransportService implements TransportService {
} }
} }
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, msg, callback); sendToDeviceActor(sessionInfo, msg, callback);
} }
} }
@ -595,7 +582,7 @@ public class DefaultTransportService implements TransportService {
dataPoints += tsKv.getKvCount(); dataPoints += tsKv.getKvCount();
} }
if (checkLimits(sessionInfo, msg, callback, dataPoints)) { if (checkLimits(sessionInfo, msg, callback, dataPoints)) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
TenantId tenantId = getTenantId(sessionInfo); TenantId tenantId = getTenantId(sessionInfo);
DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()));
CustomerId customerId = getCustomerId(sessionInfo); CustomerId customerId = getCustomerId(sessionInfo);
@ -619,7 +606,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TbMsgMetaData md, TransportServiceCallback<Void> callback) { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.PostAttributeMsg msg, TbMsgMetaData md, TransportServiceCallback<Void> callback) {
if (checkLimits(sessionInfo, msg, callback, msg.getKvCount())) { if (checkLimits(sessionInfo, msg, callback, msg.getKvCount())) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
TenantId tenantId = getTenantId(sessionInfo); TenantId tenantId = getTenantId(sessionInfo);
DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); DeviceId deviceId = new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()));
JsonObject json = JsonUtils.getJsonObject(msg.getKvList()); JsonObject json = JsonUtils.getJsonObject(msg.getKvList());
@ -639,7 +626,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetAttributeRequestMsg msg, TransportServiceCallback<Void> callback) {
if (checkLimits(sessionInfo, msg, callback)) { if (checkLimits(sessionInfo, msg, callback)) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
.setGetAttributes(msg).build(), new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback)); .setGetAttributes(msg).build(), new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback));
} }
@ -652,7 +639,7 @@ public class DefaultTransportService implements TransportService {
if (sessionMetaData != null) { if (sessionMetaData != null) {
sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe()); sessionMetaData.setSubscribedToAttributes(!msg.getUnsubscribe());
} }
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(), sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToAttributes(msg).build(),
new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback)); new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback));
} }
@ -665,7 +652,7 @@ public class DefaultTransportService implements TransportService {
if (sessionMetaData != null) { if (sessionMetaData != null) {
sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe()); sessionMetaData.setSubscribedToRPC(!msg.getUnsubscribe());
} }
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(), sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setSubscribeToRPC(msg).build(),
new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback)); new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback));
} }
@ -674,7 +661,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToDeviceRpcResponseMsg msg, TransportServiceCallback<Void> callback) {
if (checkLimits(sessionInfo, msg, callback)) { if (checkLimits(sessionInfo, msg, callback)) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(), sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setToDeviceRPCCallResponse(msg).build(),
new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback)); new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, callback));
} }
@ -683,7 +670,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void notifyAboutUplink(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.UplinkNotificationMsg msg, TransportServiceCallback<Void> callback) { public void notifyAboutUplink(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.UplinkNotificationMsg msg, TransportServiceCallback<Void> callback) {
if (checkLimits(sessionInfo, msg, callback)) { if (checkLimits(sessionInfo, msg, callback)) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setUplinkNotificationMsg(msg).build(), callback); sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setUplinkNotificationMsg(msg).build(), callback);
} }
} }
@ -704,7 +691,7 @@ public class DefaultTransportService implements TransportService {
if (checkLimits(sessionInfo, responseMsg, callback)) { if (checkLimits(sessionInfo, responseMsg, callback)) {
if (reportActivity) { if (reportActivity) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
} }
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setRpcResponseStatusMsg(responseMsg).build(), sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo).setRpcResponseStatusMsg(responseMsg).build(),
new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, TransportServiceCallback.EMPTY)); new ApiStatsProxyCallback<>(getTenantId(sessionInfo), getCustomerId(sessionInfo), 1, TransportServiceCallback.EMPTY));
@ -736,7 +723,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ToServerRpcRequestMsg msg, TransportServiceCallback<Void> callback) {
if (checkLimits(sessionInfo, msg, callback)) { if (checkLimits(sessionInfo, msg, callback)) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
UUID sessionId = toSessionId(sessionInfo); UUID sessionId = toSessionId(sessionInfo);
TenantId tenantId = getTenantId(sessionInfo); TenantId tenantId = getTenantId(sessionInfo);
DeviceId deviceId = getDeviceId(sessionInfo); DeviceId deviceId = getDeviceId(sessionInfo);
@ -761,7 +748,7 @@ public class DefaultTransportService implements TransportService {
@Override @Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback<Void> callback) { public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.ClaimDeviceMsg msg, TransportServiceCallback<Void> callback) {
if (checkLimits(sessionInfo, msg, callback)) { if (checkLimits(sessionInfo, msg, callback)) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo) sendToDeviceActor(sessionInfo, TransportToDeviceActorMsg.newBuilder().setSessionInfo(sessionInfo)
.setClaimDevice(msg).build(), callback); .setClaimDevice(msg).build(), callback);
} }
@ -780,69 +767,12 @@ public class DefaultTransportService implements TransportService {
} }
@Override @Override
public void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { public void recordActivity(TransportProtos.SessionInfoProto sessionInfo) {
reportActivityInternal(sessionInfo); recordActivityInternal(sessionInfo);
} }
private void reportActivityInternal(TransportProtos.SessionInfoProto sessionInfo) { private void recordActivityInternal(TransportProtos.SessionInfoProto sessionInfo) {
UUID sessionId = toSessionId(sessionInfo); onActivity(toSessionId(sessionInfo), getCurrentTimeMillis());
SessionActivityData sessionMetaData = sessionsActivity.computeIfAbsent(sessionId, id -> new SessionActivityData(sessionInfo));
sessionMetaData.updateLastActivityTime();
}
private void checkInactivityAndReportActivity() {
long expTime = System.currentTimeMillis() - sessionInactivityTimeout;
Set<UUID> sessionsToRemove = new HashSet<>();
sessionsActivity.forEach((uuid, sessionAD) -> {
long lastActivityTime = sessionAD.getLastActivityTime();
SessionMetaData sessionMD = sessions.get(uuid);
if (sessionMD != null) {
sessionAD.setSessionInfo(sessionMD.getSessionInfo());
} else {
sessionsToRemove.add(uuid);
}
TransportProtos.SessionInfoProto sessionInfo = sessionAD.getSessionInfo();
if (sessionInfo.getGwSessionIdMSB() != 0 && sessionInfo.getGwSessionIdLSB() != 0) {
var gwSessionId = new UUID(sessionInfo.getGwSessionIdMSB(), sessionInfo.getGwSessionIdLSB());
SessionMetaData gwMetaData = sessions.get(gwSessionId);
SessionActivityData gwActivityData = sessionsActivity.get(gwSessionId);
if (gwMetaData != null && gwMetaData.isOverwriteActivityTime()) {
lastActivityTime = Math.max(gwActivityData.getLastActivityTime(), lastActivityTime);
}
}
if (lastActivityTime < expTime) {
if (sessionMD != null) {
if (log.isDebugEnabled()) {
log.debug("[{}] Session has expired due to last activity time: {}", toSessionId(sessionInfo), lastActivityTime);
}
sessions.remove(uuid);
sessionsToRemove.add(uuid);
process(sessionInfo, SESSION_EVENT_MSG_CLOSED, null);
sessionMD.getListener().onRemoteSessionCloseCommand(uuid, SESSION_CLOSE_NOTIFICATION_PROTO);
}
} else {
if (lastActivityTime > sessionAD.getLastReportedActivityTime()) {
final long lastActivityTimeFinal = lastActivityTime;
process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder()
.setAttributeSubscription(sessionMD != null && sessionMD.isSubscribedToAttributes())
.setRpcSubscription(sessionMD != null && sessionMD.isSubscribedToRPC())
.setLastActivityTime(lastActivityTime).build(), new TransportServiceCallback<Void>() {
@Override
public void onSuccess(Void msg) {
sessionAD.setLastReportedActivityTime(lastActivityTimeFinal);
}
@Override
public void onError(Throwable e) {
log.warn("[{}] Failed to report last activity time", uuid, e);
}
});
}
}
});
// Removes all closed or short-lived sessions.
sessionsToRemove.forEach(sessionsActivity::remove);
} }
@Override @Override
@ -913,7 +843,7 @@ public class DefaultTransportService implements TransportService {
} }
TransportProtos.PostTelemetryMsg.Builder request = TransportProtos.PostTelemetryMsg.newBuilder(); TransportProtos.PostTelemetryMsg.Builder request = TransportProtos.PostTelemetryMsg.newBuilder();
TransportProtos.TsKvListProto.Builder builder = TransportProtos.TsKvListProto.newBuilder(); TransportProtos.TsKvListProto.Builder builder = TransportProtos.TsKvListProto.newBuilder();
builder.setTs(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) * 1000L + (atomicTs.getAndIncrement() % 1000)); builder.setTs(TimeUnit.MILLISECONDS.toSeconds(getCurrentTimeMillis()) * 1000L + (atomicTs.getAndIncrement() % 1000));
builder.addKv(TransportProtos.KeyValueProto.newBuilder() builder.addKv(TransportProtos.KeyValueProto.newBuilder()
.setKey("transportLog") .setKey("transportLog")
.setType(TransportProtos.KeyValueType.STRING_V) .setType(TransportProtos.KeyValueType.STRING_V)
@ -1160,12 +1090,6 @@ public class DefaultTransportService implements TransportService {
return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB())); return new DeviceId(new UUID(sessionInfo.getDeviceIdMSB(), sessionInfo.getDeviceIdLSB()));
} }
private static TransportProtos.SessionEventMsg getSessionEventMsg(TransportProtos.SessionEvent event) {
return TransportProtos.SessionEventMsg.newBuilder()
.setSessionType(TransportProtos.SessionType.ASYNC)
.setEvent(event).build();
}
protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback<Void> callback) { protected void sendToDeviceActor(TransportProtos.SessionInfoProto sessionInfo, TransportToDeviceActorMsg toDeviceActorMsg, TransportServiceCallback<Void> callback) {
ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build(); ToCoreMsg toCoreMsg = ToCoreMsg.newBuilder().setToDeviceActorMsg(toDeviceActorMsg).build();
sendToCore(getTenantId(sessionInfo), getDeviceId(sessionInfo), toCoreMsg, getRoutingKey(sessionInfo), callback); sendToCore(getTenantId(sessionInfo), getDeviceId(sessionInfo), toCoreMsg, getRoutingKey(sessionInfo), callback);

149
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportActivityManager.java

@ -0,0 +1,149 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.service;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.server.common.transport.TransportService;
import org.thingsboard.server.common.transport.TransportServiceCallback;
import org.thingsboard.server.common.transport.activity.AbstractActivityManager;
import org.thingsboard.server.common.transport.activity.ActivityReportCallback;
import org.thingsboard.server.common.transport.activity.ActivityState;
import org.thingsboard.server.common.transport.activity.strategy.ActivityStrategy;
import org.thingsboard.server.common.transport.activity.strategy.ActivityStrategyType;
import org.thingsboard.server.gen.transport.TransportProtos;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@Slf4j
public abstract class TransportActivityManager extends AbstractActivityManager<UUID, TransportProtos.SessionInfoProto> implements TransportService {
public static final String SESSION_EXPIRED_MESSAGE = "Session has expired due to last activity time!";
public static final TransportProtos.SessionEventMsg SESSION_EVENT_MSG_CLOSED = TransportProtos.SessionEventMsg.newBuilder()
.setSessionType(TransportProtos.SessionType.ASYNC)
.setEvent(TransportProtos.SessionEvent.CLOSED).build();
public static final TransportProtos.SessionCloseNotificationProto SESSION_EXPIRED_NOTIFICATION_PROTO = TransportProtos.SessionCloseNotificationProto.newBuilder()
.setMessage(SESSION_EXPIRED_MESSAGE).build();
public final ConcurrentMap<UUID, SessionMetaData> sessions = new ConcurrentHashMap<>();
@Value("${transport.sessions.report_timeout}")
protected long sessionReportTimeout;
@Value("${transport.sessions.inactivity_timeout}")
protected long sessionInactivityTimeout;
@Value("${transport.activity.reporting_strategy:LAST}")
private ActivityStrategyType reportingStrategyType;
@Override
protected long getReportingPeriodMillis() {
return sessionReportTimeout;
}
@Override
protected ActivityState<TransportProtos.SessionInfoProto> createNewState(UUID sessionId) {
SessionMetaData session = sessions.get(sessionId);
if (session == null) {
return null;
}
ActivityState<TransportProtos.SessionInfoProto> state = new ActivityState<>();
state.setMetadata(session.getSessionInfo());
return state;
}
@Override
protected ActivityStrategy getStrategy() {
return reportingStrategyType.toStrategy();
}
@Override
protected ActivityState<TransportProtos.SessionInfoProto> updateState(UUID sessionId, ActivityState<TransportProtos.SessionInfoProto> state) {
SessionMetaData session = sessions.get(sessionId);
if (session == null) {
return null;
}
state.setMetadata(session.getSessionInfo());
var sessionInfo = state.getMetadata();
if (sessionInfo.getGwSessionIdMSB() == 0L || sessionInfo.getGwSessionIdLSB() == 0L) {
return state;
}
var gwSessionId = new UUID(sessionInfo.getGwSessionIdMSB(), sessionInfo.getGwSessionIdLSB());
SessionMetaData gwSession = sessions.get(gwSessionId);
if (gwSession == null || !gwSession.isOverwriteActivityTime()) {
return state;
}
long lastRecordedTime = state.getLastRecordedTime();
long gwLastRecordedTime = getLastRecordedTime(gwSessionId);
log.debug("Session with id: [{}] has gateway session with id: [{}] with overwrite activity time enabled. " +
"Updating last activity time. Session last recorded time: [{}], gateway session last recorded time: [{}].",
sessionId, gwSessionId, lastRecordedTime, gwLastRecordedTime);
state.setLastRecordedTime(Math.max(lastRecordedTime, gwLastRecordedTime));
return state;
}
@Override
protected boolean hasExpired(long lastRecordedTime) {
return (getCurrentTimeMillis() - sessionInactivityTimeout) > lastRecordedTime;
}
@Override
protected void onStateExpiry(UUID sessionId, TransportProtos.SessionInfoProto sessionInfo) {
log.debug("Session with id: [{}] has expired due to last activity time.", sessionId);
SessionMetaData expiredSession = sessions.remove(sessionId);
if (expiredSession != null) {
deregisterSession(sessionInfo);
process(sessionInfo, SESSION_EVENT_MSG_CLOSED, null);
expiredSession.getListener().onRemoteSessionCloseCommand(sessionId, SESSION_EXPIRED_NOTIFICATION_PROTO);
}
}
@Override
protected void reportActivity(UUID sessionId, TransportProtos.SessionInfoProto currentSessionInfo, long timeToReport, ActivityReportCallback<UUID> callback) {
log.debug("Reporting activity state for session with id: [{}]. Time to report: [{}].", sessionId, timeToReport);
SessionMetaData session = sessions.get(sessionId);
TransportProtos.SubscriptionInfoProto subscriptionInfo = TransportProtos.SubscriptionInfoProto.newBuilder()
.setAttributeSubscription(session != null && session.isSubscribedToAttributes())
.setRpcSubscription(session != null && session.isSubscribedToRPC())
.setLastActivityTime(timeToReport)
.build();
TransportProtos.SessionInfoProto sessionInfo = session != null ? session.getSessionInfo() : currentSessionInfo;
process(sessionInfo, subscriptionInfo, new TransportServiceCallback<>() {
@Override
public void onSuccess(Void msgAcknowledged) {
callback.onSuccess(sessionId, timeToReport);
}
@Override
public void onError(Throwable e) {
callback.onFailure(sessionId, e);
}
});
}
protected long getCurrentTimeMillis() {
return System.currentTimeMillis();
}
}

44
common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/ActivityStrategyTypeTest.java

@ -0,0 +1,44 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
public class ActivityStrategyTypeTest {
@Test
public void testCreateAllEventsStrategy() {
assertThat(ActivityStrategyType.ALL.toStrategy()).isEqualTo(AllEventsActivityStrategy.getInstance());
}
@Test
public void testCreateFirstEventStrategy() {
assertThat(ActivityStrategyType.FIRST.toStrategy()).isEqualTo(new FirstEventActivityStrategy());
}
@Test
public void testCreateLastEventStrategy() {
assertThat(ActivityStrategyType.LAST.toStrategy()).isEqualTo(LastEventActivityStrategy.getInstance());
}
@Test
public void testCreateFirstAndLastEventStrategy() {
assertThat(ActivityStrategyType.FIRST_AND_LAST.toStrategy()).isEqualTo(new FirstAndLastEventActivityStrategy());
}
}

42
common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/AllEventsActivityStrategyTest.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class AllEventsActivityStrategyTest {
private AllEventsActivityStrategy strategy;
@BeforeEach
public void setUp() {
strategy = AllEventsActivityStrategy.getInstance();
}
@Test
public void testOnActivity() {
assertTrue(strategy.onActivity(), "onActivity() should always return true.");
}
@Test
public void testOnReportingPeriodEnd() {
assertTrue(strategy.onReportingPeriodEnd(), "onReportingPeriodEnd() should always return true.");
}
}

53
common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/FirstAndLastEventActivityStrategyTest.java

@ -0,0 +1,53 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class FirstAndLastEventActivityStrategyTest {
private FirstAndLastEventActivityStrategy strategy;
@BeforeEach
public void setUp() {
strategy = new FirstAndLastEventActivityStrategy();
}
@Test
public void testOnActivity_FirstCall() {
assertTrue(strategy.onActivity(), "First call of onActivity() should return true.");
}
@Test
public void testOnActivity_SubsequentCalls() {
assertTrue(strategy.onActivity(), "First call of onActivity() should return true.");
assertFalse(strategy.onActivity(), "Subsequent calls of onActivity() should return false.");
}
@Test
public void testOnReportingPeriodEnd() {
assertTrue(strategy.onActivity(), "First call of onActivity() should return true.");
assertTrue(strategy.onReportingPeriodEnd(), "onReportingPeriodEnd() should always return true.");
assertTrue(strategy.onActivity(), "onActivity() should return true after onReportingPeriodEnd() for the next reporting period");
assertTrue(strategy.onReportingPeriodEnd(), "onReportingPeriodEnd() should always return true.");
}
}

52
common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/FirstEventActivityStrategyTest.java

@ -0,0 +1,52 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class FirstEventActivityStrategyTest {
private FirstEventActivityStrategy strategy;
@BeforeEach
public void setUp() {
strategy = new FirstEventActivityStrategy();
}
@Test
public void testOnActivity_FirstCall() {
assertTrue(strategy.onActivity(), "First call of onActivity() should return true.");
}
@Test
public void testOnActivity_SubsequentCalls() {
assertTrue(strategy.onActivity(), "First call of onActivity() should return true.");
assertFalse(strategy.onActivity(), "Subsequent calls of onActivity() should return false.");
}
@Test
public void testOnReportingPeriodEnd() {
assertTrue(strategy.onActivity(), "First call of onActivity() should return true.");
assertFalse(strategy.onReportingPeriodEnd(), "onReportingPeriodEnd() should always return false.");
assertTrue(strategy.onActivity(), "onActivity() should return true after onReportingPeriodEnd().");
assertFalse(strategy.onReportingPeriodEnd(), "onReportingPeriodEnd() should always return false.");
}
}

43
common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/activity/strategy/LastEventActivityStrategyTest.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.activity.strategy;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class LastEventActivityStrategyTest {
private LastEventActivityStrategy strategy;
@BeforeEach
public void setUp() {
strategy = LastEventActivityStrategy.getInstance();
}
@Test
public void testOnActivity() {
assertFalse(strategy.onActivity(), "onActivity() should always return false.");
}
@Test
public void testOnReportingPeriodEnd() {
assertTrue(strategy.onReportingPeriodEnd(), "onReportingPeriodEnd() should always return true.");
}
}

508
common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/service/TransportActivityManagerTest.java

@ -0,0 +1,508 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.transport.service;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.util.ReflectionTestUtils;
import org.thingsboard.server.common.transport.SessionMsgListener;
import org.thingsboard.server.common.transport.TransportServiceCallback;
import org.thingsboard.server.common.transport.activity.ActivityReportCallback;
import org.thingsboard.server.common.transport.activity.ActivityState;
import org.thingsboard.server.common.transport.activity.strategy.ActivityStrategy;
import org.thingsboard.server.common.transport.activity.strategy.ActivityStrategyType;
import org.thingsboard.server.gen.transport.TransportProtos;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.thingsboard.server.common.transport.service.DefaultTransportService.SESSION_EVENT_MSG_CLOSED;
import static org.thingsboard.server.common.transport.service.DefaultTransportService.SESSION_EXPIRED_NOTIFICATION_PROTO;
@ExtendWith(MockitoExtension.class)
public class TransportActivityManagerTest {
private final UUID SESSION_ID = UUID.fromString("1306648a-9b26-11ee-b9d1-0242ac120002");
@Mock
private DefaultTransportService transportServiceMock;
private ConcurrentMap<UUID, SessionMetaData> sessions;
@BeforeEach
public void setup() {
sessions = new ConcurrentHashMap<>();
ReflectionTestUtils.setField(transportServiceMock, "sessions", sessions);
}
@Test
void givenKeyAndTimeToReportAndSessionExists_whenReportingActivity_thenShouldReportActivityWithSubscriptionsAndSessionInfoFromSession() {
// GIVEN
long expectedTime = 123L;
boolean expectedAttributesSubscription = true;
boolean expectedRPCSubscription = true;
TransportProtos.SessionInfoProto expectedSessionInfo = TransportProtos.SessionInfoProto.getDefaultInstance();
SessionMsgListener listenerMock = mock(SessionMsgListener.class);
SessionMetaData session = new SessionMetaData(expectedSessionInfo, TransportProtos.SessionType.ASYNC, listenerMock);
session.setSubscribedToAttributes(expectedAttributesSubscription);
session.setSubscribedToRPC(expectedRPCSubscription);
sessions.put(SESSION_ID, session);
ActivityReportCallback<UUID> callbackMock = mock(ActivityReportCallback.class);
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
doCallRealMethod().when(transportServiceMock).reportActivity(SESSION_ID, sessionInfo, expectedTime, callbackMock);
// WHEN
transportServiceMock.reportActivity(SESSION_ID, sessionInfo, expectedTime, callbackMock);
// THEN
ArgumentCaptor<TransportProtos.SessionInfoProto> sessionInfoCaptor = ArgumentCaptor.forClass(TransportProtos.SessionInfoProto.class);
ArgumentCaptor<TransportProtos.SubscriptionInfoProto> subscriptionInfoCaptor = ArgumentCaptor.forClass(TransportProtos.SubscriptionInfoProto.class);
ArgumentCaptor<TransportServiceCallback<Void>> callbackCaptor = ArgumentCaptor.forClass(TransportServiceCallback.class);
verify(transportServiceMock).process(sessionInfoCaptor.capture(), subscriptionInfoCaptor.capture(), callbackCaptor.capture());
assertThat(sessionInfoCaptor.getValue()).isEqualTo(expectedSessionInfo);
TransportProtos.SubscriptionInfoProto expectedSubscriptionInfo = TransportProtos.SubscriptionInfoProto.newBuilder()
.setAttributeSubscription(expectedAttributesSubscription)
.setRpcSubscription(expectedRPCSubscription)
.setLastActivityTime(expectedTime)
.build();
assertThat(subscriptionInfoCaptor.getValue()).isEqualTo(expectedSubscriptionInfo);
TransportServiceCallback<Void> queueCallback = callbackCaptor.getValue();
queueCallback.onSuccess(null);
verify(callbackMock).onSuccess(SESSION_ID, expectedTime);
var throwable = new Throwable();
queueCallback.onError(throwable);
verify(callbackMock).onFailure(SESSION_ID, throwable);
}
@Test
void givenKeyAndTimeToReportAndSessionDoesNotExist_whenReportingActivity_thenShouldReportActivityWithNoSubscriptionsAndPreviousSessionInfo() {
// GIVEN
long expectedTime = 123L;
boolean expectedAttributesSubscription = false;
boolean expectedRPCSubscription = false;
TransportProtos.SessionInfoProto expectedSessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
ActivityReportCallback<UUID> callbackMock = mock(ActivityReportCallback.class);
doCallRealMethod().when(transportServiceMock).reportActivity(SESSION_ID, expectedSessionInfo, expectedTime, callbackMock);
// WHEN
transportServiceMock.reportActivity(SESSION_ID, expectedSessionInfo, expectedTime, callbackMock);
// THEN
ArgumentCaptor<TransportProtos.SessionInfoProto> sessionInfoCaptor = ArgumentCaptor.forClass(TransportProtos.SessionInfoProto.class);
ArgumentCaptor<TransportProtos.SubscriptionInfoProto> subscriptionInfoCaptor = ArgumentCaptor.forClass(TransportProtos.SubscriptionInfoProto.class);
ArgumentCaptor<TransportServiceCallback<Void>> callbackCaptor = ArgumentCaptor.forClass(TransportServiceCallback.class);
verify(transportServiceMock).process(sessionInfoCaptor.capture(), subscriptionInfoCaptor.capture(), callbackCaptor.capture());
assertThat(sessionInfoCaptor.getValue()).isEqualTo(expectedSessionInfo);
TransportProtos.SubscriptionInfoProto expectedSubscriptionInfo = TransportProtos.SubscriptionInfoProto.newBuilder()
.setAttributeSubscription(expectedAttributesSubscription)
.setRpcSubscription(expectedRPCSubscription)
.setLastActivityTime(expectedTime)
.build();
assertThat(subscriptionInfoCaptor.getValue()).isEqualTo(expectedSubscriptionInfo);
TransportServiceCallback<Void> queueCallback = callbackCaptor.getValue();
queueCallback.onSuccess(null);
verify(callbackMock).onSuccess(SESSION_ID, expectedTime);
var throwable = new Throwable();
queueCallback.onError(throwable);
verify(callbackMock).onFailure(SESSION_ID, throwable);
}
@Test
void givenActivityHappened_whenRecordActivity_thenShouldDelegateToOnActivity() {
// GIVEN
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
doCallRealMethod().when(transportServiceMock).recordActivity(sessionInfo);
when(transportServiceMock.toSessionId(sessionInfo)).thenReturn(SESSION_ID);
long expectedTime = 123L;
when(transportServiceMock.getCurrentTimeMillis()).thenReturn(expectedTime);
// WHEN
transportServiceMock.recordActivity(sessionInfo);
// THEN
verify(transportServiceMock).onActivity(SESSION_ID, expectedTime);
}
@Test
void givenKey_whenCreatingNewState_thenShouldCorrectlyCreateNewEmptyState() {
// GIVEN
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
sessions.put(SESSION_ID, new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, null));
when(transportServiceMock.createNewState(SESSION_ID)).thenCallRealMethod();
ActivityState<TransportProtos.SessionInfoProto> expectedState = new ActivityState<>();
expectedState.setMetadata(sessionInfo);
// WHEN
ActivityState<TransportProtos.SessionInfoProto> actualState = transportServiceMock.createNewState(SESSION_ID);
// THEN
assertThat(actualState).isEqualTo(expectedState);
}
@ParameterizedTest
@EnumSource(ActivityStrategyType.class)
void givenDifferentReportingStrategies_whenGettingStrategy_thenShouldReturnCorrectStrategy(ActivityStrategyType reportingStrategyType) {
// GIVEN
doCallRealMethod().when(transportServiceMock).getStrategy();
ReflectionTestUtils.setField(transportServiceMock, "reportingStrategyType", reportingStrategyType);
// WHEN
ActivityStrategy actualStrategy = transportServiceMock.getStrategy();
// THEN
assertThat(actualStrategy).isEqualTo(reportingStrategyType.toStrategy());
}
@Test
void givenSessionDoesNotExist_whenUpdatingActivityState_thenShouldReturnNull() {
// GIVEN
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
ActivityState<TransportProtos.SessionInfoProto> state = new ActivityState<>();
state.setLastRecordedTime(123L);
state.setMetadata(sessionInfo);
when(transportServiceMock.updateState(SESSION_ID, state)).thenCallRealMethod();
// WHEN
ActivityState<TransportProtos.SessionInfoProto> updatedState = transportServiceMock.updateState(SESSION_ID, state);
// THEN
assertThat(updatedState).isNull();
}
@Test
void givenNoGwSessionId_whenUpdatingActivityState_thenShouldReturnSameInstanceWithUpdatedSessionInfo() {
// GIVEN
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
SessionMsgListener listenerMock = mock(SessionMsgListener.class);
sessions.put(SESSION_ID, new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listenerMock));
long lastRecordedTime = 123L;
ActivityState<TransportProtos.SessionInfoProto> state = new ActivityState<>();
state.setLastRecordedTime(lastRecordedTime);
state.setMetadata(TransportProtos.SessionInfoProto.getDefaultInstance());
when(transportServiceMock.updateState(SESSION_ID, state)).thenCallRealMethod();
// WHEN
ActivityState<TransportProtos.SessionInfoProto> updatedState = transportServiceMock.updateState(SESSION_ID, state);
// THEN
assertThat(updatedState).isSameAs(state);
assertThat(updatedState.getLastRecordedTime()).isEqualTo(lastRecordedTime);
assertThat(updatedState.getMetadata()).isEqualTo(sessionInfo);
}
@Test
void givenHasGwSessionIdButGwSessionIsNotNull_whenUpdatingActivityState_thenShouldReturnSameInstanceWithUpdatedSessionInfo() {
// GIVEN
var gwSessionId = UUID.fromString("19864038-9b48-11ee-b9d1-0242ac120002");
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.setGwSessionIdMSB(gwSessionId.getMostSignificantBits())
.setGwSessionIdLSB(gwSessionId.getLeastSignificantBits())
.build();
SessionMsgListener listenerMock = mock(SessionMsgListener.class);
sessions.put(SESSION_ID, new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listenerMock));
long lastRecordedTime = 123L;
ActivityState<TransportProtos.SessionInfoProto> state = new ActivityState<>();
state.setLastRecordedTime(lastRecordedTime);
state.setMetadata(TransportProtos.SessionInfoProto.getDefaultInstance());
when(transportServiceMock.updateState(SESSION_ID, state)).thenCallRealMethod();
// WHEN
ActivityState<TransportProtos.SessionInfoProto> updatedState = transportServiceMock.updateState(SESSION_ID, state);
// THEN
assertThat(updatedState).isSameAs(state);
assertThat(updatedState.getLastRecordedTime()).isEqualTo(lastRecordedTime);
assertThat(updatedState.getMetadata()).isEqualTo(sessionInfo);
verify(transportServiceMock, never()).getLastRecordedTime(gwSessionId);
}
@Test
void givenHasGwSessionWithoutOverwriteEnabled_whenUpdatingActivityState_thenShouldReturnSameInstanceWithUpdatedSessionInfo() {
// GIVEN
var gwSessionId = UUID.fromString("19864038-9b48-11ee-b9d1-0242ac120002");
TransportProtos.SessionInfoProto gwSessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(gwSessionId.getMostSignificantBits())
.setSessionIdLSB(gwSessionId.getLeastSignificantBits())
.build();
SessionMsgListener gwListenerMock = mock(SessionMsgListener.class);
sessions.put(gwSessionId, new SessionMetaData(gwSessionInfo, TransportProtos.SessionType.ASYNC, gwListenerMock));
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.setGwSessionIdMSB(gwSessionId.getMostSignificantBits())
.setGwSessionIdLSB(gwSessionId.getLeastSignificantBits())
.build();
SessionMsgListener listenerMock = mock(SessionMsgListener.class);
sessions.put(SESSION_ID, new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listenerMock));
long lastRecordedTime = 123L;
ActivityState<TransportProtos.SessionInfoProto> state = new ActivityState<>();
state.setLastRecordedTime(lastRecordedTime);
state.setMetadata(TransportProtos.SessionInfoProto.getDefaultInstance());
when(transportServiceMock.updateState(SESSION_ID, state)).thenCallRealMethod();
// WHEN
ActivityState<TransportProtos.SessionInfoProto> updatedState = transportServiceMock.updateState(SESSION_ID, state);
// THEN
assertThat(updatedState).isSameAs(state);
assertThat(updatedState.getLastRecordedTime()).isEqualTo(lastRecordedTime);
assertThat(updatedState.getMetadata()).isEqualTo(sessionInfo);
verify(transportServiceMock, never()).getLastRecordedTime(gwSessionId);
}
@Test
void givenHasGwSessionWithOverwriteEnabledAndGwLastRecordedTimeIsGreater_whenUpdatingActivityState_thenShouldReturnSameInstanceWithUpdatedSessionInfoAndLastRecordedTime() {
// GIVEN
var gwSessionId = UUID.fromString("19864038-9b48-11ee-b9d1-0242ac120002");
TransportProtos.SessionInfoProto gwSessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(gwSessionId.getMostSignificantBits())
.setSessionIdLSB(gwSessionId.getLeastSignificantBits())
.build();
SessionMsgListener gwListenerMock = mock(SessionMsgListener.class);
SessionMetaData gwSession = new SessionMetaData(gwSessionInfo, TransportProtos.SessionType.ASYNC, gwListenerMock);
gwSession.setOverwriteActivityTime(true);
sessions.put(gwSessionId, gwSession);
long gwLastRecordedTime = 500L;
when(transportServiceMock.getLastRecordedTime(gwSessionId)).thenReturn(gwLastRecordedTime);
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.setGwSessionIdMSB(gwSessionId.getMostSignificantBits())
.setGwSessionIdLSB(gwSessionId.getLeastSignificantBits())
.build();
SessionMsgListener listenerMock = mock(SessionMsgListener.class);
sessions.put(SESSION_ID, new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listenerMock));
long lastRecordedTime = 123L;
ActivityState<TransportProtos.SessionInfoProto> state = new ActivityState<>();
state.setLastRecordedTime(lastRecordedTime);
state.setMetadata(TransportProtos.SessionInfoProto.getDefaultInstance());
when(transportServiceMock.updateState(SESSION_ID, state)).thenCallRealMethod();
// WHEN
ActivityState<TransportProtos.SessionInfoProto> updatedState = transportServiceMock.updateState(SESSION_ID, state);
// THEN
assertThat(updatedState).isSameAs(state);
assertThat(updatedState.getLastRecordedTime()).isEqualTo(gwLastRecordedTime);
assertThat(updatedState.getMetadata()).isEqualTo(sessionInfo);
}
@Test
void givenHasGwSessionWithOverwriteEnabledAndGwLastRecordedTimeIsLess_whenUpdatingActivityState_thenShouldReturnSameInstanceWithUpdatedSessionInfoOnly() {
// GIVEN
var gwSessionId = UUID.fromString("19864038-9b48-11ee-b9d1-0242ac120002");
TransportProtos.SessionInfoProto gwSessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(gwSessionId.getMostSignificantBits())
.setSessionIdLSB(gwSessionId.getLeastSignificantBits())
.build();
SessionMsgListener gwListenerMock = mock(SessionMsgListener.class);
SessionMetaData gwSession = new SessionMetaData(gwSessionInfo, TransportProtos.SessionType.ASYNC, gwListenerMock);
gwSession.setOverwriteActivityTime(true);
sessions.put(gwSessionId, gwSession);
long gwLastRecordedTime = 100L;
when(transportServiceMock.getLastRecordedTime(gwSessionId)).thenReturn(gwLastRecordedTime);
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.setGwSessionIdMSB(gwSessionId.getMostSignificantBits())
.setGwSessionIdLSB(gwSessionId.getLeastSignificantBits())
.build();
SessionMsgListener listenerMock = mock(SessionMsgListener.class);
sessions.put(SESSION_ID, new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listenerMock));
long lastRecordedTime = 123L;
ActivityState<TransportProtos.SessionInfoProto> state = new ActivityState<>();
state.setLastRecordedTime(lastRecordedTime);
state.setMetadata(TransportProtos.SessionInfoProto.getDefaultInstance());
when(transportServiceMock.updateState(SESSION_ID, state)).thenCallRealMethod();
// WHEN
ActivityState<TransportProtos.SessionInfoProto> updatedState = transportServiceMock.updateState(SESSION_ID, state);
// THEN
assertThat(updatedState).isSameAs(state);
assertThat(updatedState.getLastRecordedTime()).isEqualTo(lastRecordedTime);
assertThat(updatedState.getMetadata()).isEqualTo(sessionInfo);
}
@ParameterizedTest
@MethodSource("provideTestParamsForHasExpiredTrue")
public void givenExpiredLastRecordedTime_whenCheckingForExpiry_thenShouldReturnTrue(long currentTimeMillis, long lastRecordedTime, long sessionInactivityTimeout) {
// GIVEN
ReflectionTestUtils.setField(transportServiceMock, "sessionInactivityTimeout", sessionInactivityTimeout);
when(transportServiceMock.getCurrentTimeMillis()).thenReturn(currentTimeMillis);
when(transportServiceMock.hasExpired(lastRecordedTime)).thenCallRealMethod();
// WHEN
boolean hasExpired = transportServiceMock.hasExpired(lastRecordedTime);
// THEN
assertThat(hasExpired).isTrue();
}
private static Stream<Arguments> provideTestParamsForHasExpiredTrue() {
return Stream.of(
Arguments.of(10L, 0L, 9L),
Arguments.of(10L, 7L, 2L),
Arguments.of(10L, 8L, 1L),
Arguments.of(10000L, 5000L, 3000L)
);
}
@ParameterizedTest
@MethodSource("provideTestParamsForHasExpiredFalse")
public void givenNotExpiredLastRecordedTime_whenCheckingForExpiry_thenShouldReturnFalse(long currentTimeMillis, long lastRecordedTime, long sessionInactivityTimeout) {
// GIVEN
ReflectionTestUtils.setField(transportServiceMock, "sessionInactivityTimeout", sessionInactivityTimeout);
when(transportServiceMock.getCurrentTimeMillis()).thenReturn(currentTimeMillis);
when(transportServiceMock.hasExpired(lastRecordedTime)).thenCallRealMethod();
// WHEN
boolean hasExpired = transportServiceMock.hasExpired(lastRecordedTime);
// THEN
assertThat(hasExpired).isFalse();
}
private static Stream<Arguments> provideTestParamsForHasExpiredFalse() {
return Stream.of(
Arguments.of(10L, 9L, 2L),
Arguments.of(10L, 0L, 11L),
Arguments.of(10L, 8L, 3L),
Arguments.of(10000L, 8000L, 3000L)
);
}
@Test
void givenSessionExists_whenOnStateExpiryCalled_thenShouldPerformExpirationActions() {
// GIVEN
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
SessionMsgListener listenerMock = mock(SessionMsgListener.class);
sessions.put(SESSION_ID, new SessionMetaData(sessionInfo, TransportProtos.SessionType.ASYNC, listenerMock));
doCallRealMethod().when(transportServiceMock).onStateExpiry(SESSION_ID, sessionInfo);
// WHEN
transportServiceMock.onStateExpiry(SESSION_ID, sessionInfo);
// THEN
assertThat(sessions.containsKey(SESSION_ID)).isFalse();
verify(transportServiceMock).deregisterSession(sessionInfo);
verify(transportServiceMock).process(sessionInfo, SESSION_EVENT_MSG_CLOSED, null);
verify(listenerMock).onRemoteSessionCloseCommand(SESSION_ID, SESSION_EXPIRED_NOTIFICATION_PROTO);
}
@Test
void givenSessionDoesNotExist_whenOnStateExpiryCalled_thenShouldNotPerformExpirationActions() {
// GIVEN
TransportProtos.SessionInfoProto sessionInfo = TransportProtos.SessionInfoProto.newBuilder()
.setSessionIdMSB(SESSION_ID.getMostSignificantBits())
.setSessionIdLSB(SESSION_ID.getLeastSignificantBits())
.build();
doCallRealMethod().when(transportServiceMock).onStateExpiry(SESSION_ID, sessionInfo);
// WHEN
transportServiceMock.onStateExpiry(SESSION_ID, sessionInfo);
// THEN
assertThat(sessions.containsKey(SESSION_ID)).isFalse();
verify(transportServiceMock, never()).deregisterSession(sessionInfo);
verify(transportServiceMock, never()).process(sessionInfo, SESSION_EVENT_MSG_CLOSED, null);
}
}

3
dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityConfiguration.java

@ -18,14 +18,15 @@ package org.thingsboard.server.dao.device;
import lombok.Data; import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@Profile("install")
@Configuration @Configuration
@ConfigurationProperties(prefix = "device") @ConfigurationProperties(prefix = "device")
@Data @Data
@Deprecated(since = "3.6.1")
public class DeviceConnectivityConfiguration { public class DeviceConnectivityConfiguration {
private Map<String, DeviceConnectivityInfo> connectivity = new HashMap<>(); private Map<String, DeviceConnectivityInfo> connectivity = new HashMap<>();

2
dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java

@ -354,6 +354,7 @@ public class ModelConstants {
public static final String COMPONENT_DESCRIPTOR_CONFIGURATION_DESCRIPTOR_PROPERTY = "configuration_descriptor"; public static final String COMPONENT_DESCRIPTOR_CONFIGURATION_DESCRIPTOR_PROPERTY = "configuration_descriptor";
public static final String COMPONENT_DESCRIPTOR_CONFIGURATION_VERSION_PROPERTY = "configuration_version"; public static final String COMPONENT_DESCRIPTOR_CONFIGURATION_VERSION_PROPERTY = "configuration_version";
public static final String COMPONENT_DESCRIPTOR_ACTIONS_PROPERTY = "actions"; public static final String COMPONENT_DESCRIPTOR_ACTIONS_PROPERTY = "actions";
public static final String COMPONENT_DESCRIPTOR_HAS_QUEUE_NAME_PROPERTY = "has_queue_name";
/** /**
* Event constants. * Event constants.
@ -389,6 +390,7 @@ public class ModelConstants {
public static final String DEBUG_MODE = "debug_mode"; public static final String DEBUG_MODE = "debug_mode";
public static final String SINGLETON_MODE = "singleton_mode"; public static final String SINGLETON_MODE = "singleton_mode";
public static final String QUEUE_NAME = "queue_name";
/** /**
* Rule chain constants. * Rule chain constants.

5
dao/src/main/java/org/thingsboard/server/dao/model/sql/ComponentDescriptorEntity.java

@ -70,6 +70,9 @@ public class ComponentDescriptorEntity extends BaseSqlEntity<ComponentDescriptor
@Column(name = ModelConstants.COMPONENT_DESCRIPTOR_ACTIONS_PROPERTY) @Column(name = ModelConstants.COMPONENT_DESCRIPTOR_ACTIONS_PROPERTY)
private String actions; private String actions;
@Column(name = ModelConstants.COMPONENT_DESCRIPTOR_HAS_QUEUE_NAME_PROPERTY)
private boolean hasQueueName;
public ComponentDescriptorEntity() { public ComponentDescriptorEntity() {
} }
@ -86,6 +89,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity<ComponentDescriptor
this.clazz = component.getClazz(); this.clazz = component.getClazz();
this.configurationDescriptor = component.getConfigurationDescriptor(); this.configurationDescriptor = component.getConfigurationDescriptor();
this.configurationVersion = component.getConfigurationVersion(); this.configurationVersion = component.getConfigurationVersion();
this.hasQueueName = component.isHasQueueName();
} }
@Override @Override
@ -100,6 +104,7 @@ public class ComponentDescriptorEntity extends BaseSqlEntity<ComponentDescriptor
data.setActions(this.getActions()); data.setActions(this.getActions());
data.setConfigurationDescriptor(configurationDescriptor); data.setConfigurationDescriptor(configurationDescriptor);
data.setConfigurationVersion(configurationVersion); data.setConfigurationVersion(configurationVersion);
data.setHasQueueName(hasQueueName);
return data; return data;
} }
} }

5
dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java

@ -66,6 +66,9 @@ public class RuleNodeEntity extends BaseSqlEntity<RuleNode> {
@Column(name = ModelConstants.SINGLETON_MODE) @Column(name = ModelConstants.SINGLETON_MODE)
private boolean singletonMode; private boolean singletonMode;
@Column(name = ModelConstants.QUEUE_NAME)
private String queueName;
@Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY)
private UUID externalId; private UUID externalId;
@ -84,6 +87,7 @@ public class RuleNodeEntity extends BaseSqlEntity<RuleNode> {
this.name = ruleNode.getName(); this.name = ruleNode.getName();
this.debugMode = ruleNode.isDebugMode(); this.debugMode = ruleNode.isDebugMode();
this.singletonMode = ruleNode.isSingletonMode(); this.singletonMode = ruleNode.isSingletonMode();
this.queueName = ruleNode.getQueueName();
this.configurationVersion = ruleNode.getConfigurationVersion(); this.configurationVersion = ruleNode.getConfigurationVersion();
this.configuration = ruleNode.getConfiguration(); this.configuration = ruleNode.getConfiguration();
this.additionalInfo = ruleNode.getAdditionalInfo(); this.additionalInfo = ruleNode.getAdditionalInfo();
@ -103,6 +107,7 @@ public class RuleNodeEntity extends BaseSqlEntity<RuleNode> {
ruleNode.setName(name); ruleNode.setName(name);
ruleNode.setDebugMode(debugMode); ruleNode.setDebugMode(debugMode);
ruleNode.setSingletonMode(singletonMode); ruleNode.setSingletonMode(singletonMode);
ruleNode.setQueueName(queueName);
ruleNode.setConfigurationVersion(configurationVersion); ruleNode.setConfigurationVersion(configurationVersion);
ruleNode.setConfiguration(configuration); ruleNode.setConfiguration(configuration);
ruleNode.setAdditionalInfo(additionalInfo); ruleNode.setAdditionalInfo(additionalInfo);

3
dao/src/main/java/org/thingsboard/server/dao/sql/component/AbstractComponentDescriptorInsertRepository.java

@ -77,7 +77,8 @@ public abstract class AbstractComponentDescriptorInsertRepository implements Com
.setParameter("name", entity.getName()) .setParameter("name", entity.getName())
.setParameter("scope", entity.getScope().name()) .setParameter("scope", entity.getScope().name())
.setParameter("type", entity.getType().name()) .setParameter("type", entity.getType().name())
.setParameter("clustering_mode", entity.getClusteringMode().name()); .setParameter("clustering_mode", entity.getClusteringMode().name())
.setParameter("has_queue_name", entity.isHasQueueName());
} }
private ComponentDescriptorEntity processSaveOrUpdate(ComponentDescriptorEntity entity, String query) { private ComponentDescriptorEntity processSaveOrUpdate(ComponentDescriptorEntity entity, String query) {

4
dao/src/main/java/org/thingsboard/server/dao/sql/component/SqlComponentDescriptorInsertRepository.java

@ -44,10 +44,10 @@ public class SqlComponentDescriptorInsertRepository extends AbstractComponentDes
} }
private static String getInsertOrUpdateStatement(String conflictKeyStatement, String updateKeyStatement) { private static String getInsertOrUpdateStatement(String conflictKeyStatement, String updateKeyStatement) {
return "INSERT INTO component_descriptor (id, created_time, actions, clazz, configuration_descriptor, configuration_version, name, scope, type, clustering_mode) VALUES (:id, :created_time, :actions, :clazz, :configuration_descriptor, :configuration_version, :name, :scope, :type, :clustering_mode) ON CONFLICT " + conflictKeyStatement + " DO UPDATE SET " + updateKeyStatement + " returning *"; return "INSERT INTO component_descriptor (id, created_time, actions, clazz, configuration_descriptor, configuration_version, name, scope, type, clustering_mode, has_queue_name) VALUES (:id, :created_time, :actions, :clazz, :configuration_descriptor, :configuration_version, :name, :scope, :type, :clustering_mode, :has_queue_name) ON CONFLICT " + conflictKeyStatement + " DO UPDATE SET " + updateKeyStatement + " returning *";
} }
private static String getUpdateStatement(String id) { private static String getUpdateStatement(String id) {
return "actions = :actions, " + id + ",created_time = :created_time, configuration_descriptor = :configuration_descriptor, configuration_version = :configuration_version, name = :name, scope = :scope, type = :type, clustering_mode = :clustering_mode"; return "actions = :actions, " + id + ",created_time = :created_time, configuration_descriptor = :configuration_descriptor, configuration_version = :configuration_version, name = :name, scope = :scope, type = :type, clustering_mode = :clustering_mode, has_queue_name = :has_queue_name";
} }
} }

23
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java

@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.IntervalType;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult;
import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry;
@ -37,9 +38,18 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository;
import org.thingsboard.server.dao.sqlts.ts.TsKvRepository; import org.thingsboard.server.dao.sqlts.ts.TsKvRepository;
import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.dao.timeseries.TimeseriesDao;
import org.thingsboard.server.dao.util.TimeUtils;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy; import javax.annotation.PreDestroy;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.time.temporal.IsoFields;
import java.time.temporal.TemporalUnit;
import java.time.temporal.WeekFields;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
@ -112,16 +122,23 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
@Override @Override
public ListenableFuture<ReadTsKvQueryResult> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { public ListenableFuture<ReadTsKvQueryResult> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
if (query.getAggregation() == Aggregation.NONE) { var aggParams = query.getAggParameters();
if (Aggregation.NONE.equals(aggParams.getAggregation())) {
return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query));
} else { } else {
List<ListenableFuture<Optional<TsKvEntity>>> futures = new ArrayList<>(); List<ListenableFuture<Optional<TsKvEntity>>> futures = new ArrayList<>();
var intervalType = aggParams.getIntervalType();
long startPeriod = query.getStartTs(); long startPeriod = query.getStartTs();
long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs());
long step = query.getInterval();
while (startPeriod < endPeriod) { while (startPeriod < endPeriod) {
long startTs = startPeriod; long startTs = startPeriod;
long endTs = Math.min(startPeriod + step, endPeriod); long endTs;
if (IntervalType.MILLISECONDS.equals(intervalType)) {
endTs = startPeriod + aggParams.getInterval();
} else {
endTs = TimeUtils.calculateIntervalEnd(startTs, intervalType, aggParams.getTzId());
}
endTs = Math.min(endTs, endPeriod);
long ts = startTs + (endTs - startTs) / 2; long ts = startTs + (endTs - startTs) / 2;
ListenableFuture<Optional<TsKvEntity>> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); ListenableFuture<Optional<TsKvEntity>> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation());
futures.add(aggregateTsKvEntry); futures.add(aggregateTsKvEntry);

24
dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java

@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort;
@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.IntervalType;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult;
import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry;
@ -35,11 +37,13 @@ import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity;
import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity;
import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao;
import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository;
import org.thingsboard.server.dao.timeseries.TimeseriesDao; import org.thingsboard.server.dao.timeseries.TimeseriesDao;
import org.thingsboard.server.dao.util.TimeUtils;
import org.thingsboard.server.dao.util.TimescaleDBTsDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
@ -143,14 +147,28 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
@Override @Override
public ListenableFuture<ReadTsKvQueryResult> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { public ListenableFuture<ReadTsKvQueryResult> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
var aggParams = query.getAggParameters();
var intervalType = aggParams.getIntervalType();
if (query.getAggregation() == Aggregation.NONE) { if (query.getAggregation() == Aggregation.NONE) {
return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query)); return Futures.immediateFuture(findAllAsyncWithLimit(entityId, query));
} else { } else if (IntervalType.MILLISECONDS.equals(intervalType)) {
long startTs = query.getStartTs(); long startTs = query.getStartTs();
long endTs = Math.max(query.getStartTs() + 1, query.getEndTs()); long endTs = Math.max(query.getStartTs() + 1, query.getEndTs());
long timeBucket = query.getInterval(); long timeBucket = query.getInterval();
List<Optional<? extends AbstractTsKvEntity>> data = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation()); List<Optional<? extends AbstractTsKvEntity>> data = findAllAndAggregateAsync(entityId, query.getKey(), startTs, endTs, timeBucket, query.getAggregation());
return getReadTsKvQueryResultFuture(query, Futures.immediateFuture(data)); return getReadTsKvQueryResultFuture(query, Futures.immediateFuture(data));
} else {
//TODO: @dshvaika improve according to native capabilities of Timescale.
long startPeriod = query.getStartTs();
long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs());
List<TimescaleTsKvEntity> timescaleTsKvEntities = new ArrayList<>();
while (startPeriod < endPeriod) {
long startTs = startPeriod;
long endTs = Math.min(TimeUtils.calculateIntervalEnd(startTs, intervalType, aggParams.getTzId()), endPeriod);
timescaleTsKvEntities.addAll(switchAggregation(query.getKey(), startTs, endTs, endTs - startTs, query.getAggregation(), entityId.getId()));
startPeriod = endTs;
}
return getReadTsKvQueryResultFuture(query, Futures.immediateFuture(toResultList(entityId, query.getKey(), timescaleTsKvEntities)));
} }
} }
@ -187,6 +205,10 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements
timescaleTsKvEntities.addAll(switchAggregation(key, startTs + interval, endTs, remainingPart, aggregation, entityId.getId())); timescaleTsKvEntities.addAll(switchAggregation(key, startTs + interval, endTs, remainingPart, aggregation, entityId.getId()));
} }
return toResultList(entityId, key, timescaleTsKvEntities);
}
private static List<Optional<? extends AbstractTsKvEntity>> toResultList(EntityId entityId, String key, List<TimescaleTsKvEntity> timescaleTsKvEntities) {
if (!CollectionUtils.isEmpty(timescaleTsKvEntities)) { if (!CollectionUtils.isEmpty(timescaleTsKvEntities)) {
List<Optional<? extends AbstractTsKvEntity>> result = new ArrayList<>(); List<Optional<? extends AbstractTsKvEntity>> result = new ArrayList<>();
timescaleTsKvEntities.forEach(entity -> { timescaleTsKvEntities.forEach(entity -> {

18
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DataType;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.IntervalType;
import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult;
@ -51,6 +52,7 @@ import org.thingsboard.server.dao.nosql.TbResultSet;
import org.thingsboard.server.dao.nosql.TbResultSetFuture; import org.thingsboard.server.dao.nosql.TbResultSetFuture;
import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao;
import org.thingsboard.server.dao.util.NoSqlTsDao; import org.thingsboard.server.dao.util.NoSqlTsDao;
import org.thingsboard.server.dao.util.TimeUtils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
@ -227,18 +229,24 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
@Override @Override
public ListenableFuture<ReadTsKvQueryResult> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { public ListenableFuture<ReadTsKvQueryResult> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) {
if (query.getAggregation() == Aggregation.NONE) { var aggParams = query.getAggParameters();
if (Aggregation.NONE.equals(aggParams.getAggregation())) {
return findAllAsyncWithLimit(tenantId, entityId, query); return findAllAsyncWithLimit(tenantId, entityId, query);
} else { } else {
long startPeriod = query.getStartTs(); long startPeriod = query.getStartTs();
long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs()); long endPeriod = Math.max(query.getStartTs() + 1, query.getEndTs());
long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS);
List<ListenableFuture<Optional<TsKvEntryAggWrapper>>> futures = new ArrayList<>(); List<ListenableFuture<Optional<TsKvEntryAggWrapper>>> futures = new ArrayList<>();
var intervalType = aggParams.getIntervalType();
while (startPeriod < endPeriod) { while (startPeriod < endPeriod) {
long startTs = startPeriod; long startTs = startPeriod;
long endTs = Math.min(startPeriod + step, endPeriod); long endTs;
long ts = endTs - startTs; if (IntervalType.MILLISECONDS.equals(intervalType)) {
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); endTs = startPeriod + Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS);
} else {
endTs = TimeUtils.calculateIntervalEnd(startTs, aggParams.getIntervalType(), aggParams.getTzId());
}
endTs = Math.min(endTs, endPeriod);
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, query.getAggregation(), query.getOrder());
futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs)));
startPeriod = endTs; startPeriod = endTs;
} }

2
dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java

@ -33,7 +33,7 @@ public class TsKvQueryCursor extends QueryCursor {
@Getter @Getter
private final List<TsKvEntry> data; private final List<TsKvEntry> data;
@Getter @Getter
private String orderBy; private final String orderBy;
private int partitionIndex; private int partitionIndex;
private int currentLimit; private int currentLimit;

45
dao/src/main/java/org/thingsboard/server/dao/util/TimeUtils.java

@ -0,0 +1,45 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.thingsboard.server.common.data.kv.IntervalType;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.time.temporal.IsoFields;
import java.time.temporal.WeekFields;
public class TimeUtils {
public static long calculateIntervalEnd(long startTs, IntervalType intervalType, ZoneId tzId) {
var startTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTs), tzId);
switch (intervalType) {
case WEEK:
return startTime.truncatedTo(ChronoUnit.DAYS).with(WeekFields.SUNDAY_START.dayOfWeek(), 1).plusDays(7).toInstant().toEpochMilli();
case WEEK_ISO:
return startTime.truncatedTo(ChronoUnit.DAYS).with(WeekFields.ISO.dayOfWeek(), 1).plusDays(7).toInstant().toEpochMilli();
case MONTH:
return startTime.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).plusMonths(1).toInstant().toEpochMilli();
case QUARTER:
return startTime.truncatedTo(ChronoUnit.DAYS).with(IsoFields.DAY_OF_QUARTER, 1).plusMonths(3).toInstant().toEpochMilli();
default:
throw new RuntimeException("Not supported!");
}
}
}

4
dao/src/main/resources/sql/schema-entities.sql

@ -126,7 +126,8 @@ CREATE TABLE IF NOT EXISTS component_descriptor (
name varchar(255), name varchar(255),
scope varchar(255), scope varchar(255),
type varchar(255), type varchar(255),
clustering_mode varchar(255) clustering_mode varchar(255),
has_queue_name boolean DEFAULT false
); );
CREATE TABLE IF NOT EXISTS customer ( CREATE TABLE IF NOT EXISTS customer (
@ -187,6 +188,7 @@ CREATE TABLE IF NOT EXISTS rule_node (
name varchar(255), name varchar(255),
debug_mode boolean, debug_mode boolean,
singleton_mode boolean, singleton_mode boolean,
queue_name varchar(255),
external_id uuid external_id uuid
); );

60
dao/src/test/java/org/thingsboard/server/dao/util/TimeUtilsTest.java

@ -0,0 +1,60 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.junit.jupiter.api.Test;
import org.thingsboard.server.common.data.kv.IntervalType;
import java.time.ZoneId;
import static org.assertj.core.api.Assertions.assertThat;
class TimeUtilsTest {
@Test
void testWeekEnd() {
long ts = 1704899727000L; // Wednesday, January 10 15:15:27 GMT
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK, ZoneId.of("Europe/Kyiv"))).isEqualTo(1705183200000L); // Sunday, January 14, 2024 0:00:00 GMT+02:00
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK_ISO, ZoneId.of("Europe/Kyiv"))).isEqualTo(1705269600000L); // Monday, January 15, 2024 0:00:00 GMT+02:00
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1705186800000L); // Sunday, January 14, 2024 0:00:00 GMT+01:00
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK_ISO, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1705273200000L); // Monday, January 15, 2024 0:00:00 GMT+01:00
ts = 1704621600000L; // Sunday, January 7, 2024 12:00:00 GMT+02:00
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK, ZoneId.of("Europe/Kyiv"))).isEqualTo(1705183200000L); // Sunday, January 14, 2024 0:00:00 GMT+02:00
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.WEEK_ISO, ZoneId.of("Europe/Kyiv"))).isEqualTo(1704664800000L); // Monday, January 8, 2024 0:00:00 GMT+02:00
}
@Test
void testMonthEnd() {
long ts = 1704899727000L; // Wednesday, January 10 15:15:27 GMT
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.MONTH, ZoneId.of("Europe/Kyiv"))).isEqualTo(1706738400000L); // Thursday, February 1, 2024 0:00:00 GMT+02:00
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.MONTH, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1706742000000L); // Monday, January 15, 2024 0:00:00 GMT+02:00
}
@Test
void testQuarterEnd() {
long ts = 1704899727000L; // Wednesday, January 10 15:15:27 GMT
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("Europe/Kyiv"))).isEqualTo(1711918800000L); // Monday, April 1, 2024 0:00:00 GMT+03:00 DST
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("Europe/Amsterdam"))).isEqualTo(1711922400000L); // Monday, April 1, 2024 1:00:00 GMT+03:00 DST
ts = 1711929600000L; // Monday, April 1, 2024 3:00:00 GMT+03:00
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("Europe/Kyiv"))).isEqualTo(1719781200000L); // Monday, July 1, 2024 0:00:00 GMT+03:00 DST
assertThat(TimeUtils.calculateIntervalEnd(ts, IntervalType.QUARTER, ZoneId.of("America/New_York"))).isEqualTo(1711944000000L); // Monday, April 1, 2024 7:00:00 GMT+03:00 DST
}
}

4
monitoring/src/main/java/org/thingsboard/monitoring/config/MonitoringTarget.java

@ -23,6 +23,10 @@ public interface MonitoringTarget {
String getBaseUrl(); String getBaseUrl();
default String getQueue() {
return "Main";
}
boolean isCheckDomainIps(); boolean isCheckDomainIps();
} }

7
monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportInfo.java

@ -22,10 +22,15 @@ public class TransportInfo {
private final TransportType transportType; private final TransportType transportType;
private final String baseUrl; private final String baseUrl;
private final String queue;
@Override @Override
public String toString() { public String toString() {
return String.format("%s transport (%s)", transportType, baseUrl); if (queue.equals("Main")) {
return String.format("*%s* (%s)", transportType.getName(), baseUrl);
} else {
return String.format("*%s* (%s) _%s_", transportType.getName(), baseUrl, queue);
}
} }
} }

6
monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportMonitoringTarget.java

@ -16,6 +16,7 @@
package org.thingsboard.monitoring.config.transport; package org.thingsboard.monitoring.config.transport;
import lombok.Data; import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.thingsboard.monitoring.config.MonitoringTarget; import org.thingsboard.monitoring.config.MonitoringTarget;
import java.util.UUID; import java.util.UUID;
@ -25,6 +26,7 @@ public class TransportMonitoringTarget implements MonitoringTarget {
private String baseUrl; private String baseUrl;
private DeviceConfig device; // set manually during initialization private DeviceConfig device; // set manually during initialization
private String queue;
private boolean checkDomainIps; private boolean checkDomainIps;
@Override @Override
@ -32,4 +34,8 @@ public class TransportMonitoringTarget implements MonitoringTarget {
return device.getId(); return device.getId();
} }
public String getQueue() {
return StringUtils.defaultIfEmpty(queue, "Main");
}
} }

9
monitoring/src/main/java/org/thingsboard/monitoring/config/transport/TransportType.java

@ -27,11 +27,12 @@ import org.thingsboard.monitoring.service.transport.impl.MqttTransportHealthChec
@Getter @Getter
public enum TransportType { public enum TransportType {
MQTT(MqttTransportHealthChecker.class), MQTT("MQTT", MqttTransportHealthChecker.class),
COAP(CoapTransportHealthChecker.class), COAP("CoAP",CoapTransportHealthChecker.class),
HTTP(HttpTransportHealthChecker.class), HTTP("HTTP", HttpTransportHealthChecker.class),
LWM2M(Lwm2mTransportHealthChecker.class); LWM2M("LwM2M", Lwm2mTransportHealthChecker.class);
private final String name;
private final Class<? extends TransportHealthChecker<?>> serviceClass; private final Class<? extends TransportHealthChecker<?>> serviceClass;
} }

2
monitoring/src/main/java/org/thingsboard/monitoring/data/notification/HighLatencyNotification.java

@ -34,7 +34,7 @@ public class HighLatencyNotification implements Notification {
StringBuilder text = new StringBuilder(); StringBuilder text = new StringBuilder();
text.append("Some of the latencies are higher than ").append(thresholdMs).append(" ms:\n"); text.append("Some of the latencies are higher than ").append(thresholdMs).append(" ms:\n");
highLatencies.forEach(latency -> { highLatencies.forEach(latency -> {
text.append(String.format("[%s] %s\n", latency.getKey(), latency.getFormattedValue())); text.append(String.format("[%s] *%s*\n", latency.getKey(), latency.getFormattedValue()));
}); });
return text.toString(); return text.toString();
} }

2
monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceFailureNotification.java

@ -43,7 +43,7 @@ public class ServiceFailureNotification implements Notification {
if (errorMsg == null) { if (errorMsg == null) {
errorMsg = error.getClass().getSimpleName(); errorMsg = error.getClass().getSimpleName();
} }
return String.format("[%s] Failure: %s (number of subsequent failures: %s)", serviceKey, errorMsg, failuresCount); return String.format("%s - Failure: %s (number of subsequent failures: %s)", serviceKey, errorMsg, failuresCount);
} }
} }

2
monitoring/src/main/java/org/thingsboard/monitoring/data/notification/ServiceRecoveryNotification.java

@ -25,7 +25,7 @@ public class ServiceRecoveryNotification implements Notification {
@Override @Override
public String getText() { public String getText() {
return String.format("[%s] is OK", serviceKey); return String.format("%s is OK", serviceKey);
} }
} }

18
monitoring/src/main/java/org/thingsboard/monitoring/notification/NotificationService.java

@ -17,6 +17,8 @@ package org.thingsboard.monitoring.notification;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.thingsboard.monitoring.data.notification.Notification; import org.thingsboard.monitoring.data.notification.Notification;
import org.thingsboard.monitoring.notification.channels.NotificationChannel; import org.thingsboard.monitoring.notification.channels.NotificationChannel;
@ -24,7 +26,6 @@ import org.thingsboard.monitoring.notification.channels.NotificationChannel;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.function.Consumer;
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
@ -34,15 +35,20 @@ public class NotificationService {
private final List<NotificationChannel> notificationChannels; private final List<NotificationChannel> notificationChannels;
private final ExecutorService notificationExecutor = Executors.newSingleThreadExecutor(); private final ExecutorService notificationExecutor = Executors.newSingleThreadExecutor();
public void sendNotification(Notification notification) { @Value("${monitoring.notifications.message_prefix}")
forEachNotificationChannel(notificationChannel -> notificationChannel.sendNotification(notification)); private String messagePrefix;
}
private void forEachNotificationChannel(Consumer<NotificationChannel> function) { public void sendNotification(Notification notification) {
String message;
if (StringUtils.isEmpty(messagePrefix)) {
message = notification.getText();
} else {
message = messagePrefix + System.lineSeparator() + notification.getText();
}
notificationChannels.forEach(notificationChannel -> { notificationChannels.forEach(notificationChannel -> {
notificationExecutor.submit(() -> { notificationExecutor.submit(() -> {
try { try {
function.accept(notificationChannel); notificationChannel.sendNotification(message);
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to send notification to {}", notificationChannel.getClass().getSimpleName(), e); log.error("Failed to send notification to {}", notificationChannel.getClass().getSimpleName(), e);
} }

4
monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/NotificationChannel.java

@ -15,10 +15,8 @@
*/ */
package org.thingsboard.monitoring.notification.channels; package org.thingsboard.monitoring.notification.channels;
import org.thingsboard.monitoring.data.notification.Notification;
public interface NotificationChannel { public interface NotificationChannel {
void sendNotification(Notification notification); void sendNotification(String message);
} }

11
monitoring/src/main/java/org/thingsboard/monitoring/notification/channels/impl/SlackNotificationChannel.java

@ -21,7 +21,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate; import org.springframework.web.client.RestTemplate;
import org.thingsboard.monitoring.data.notification.Notification;
import org.thingsboard.monitoring.notification.channels.NotificationChannel; import org.thingsboard.monitoring.notification.channels.NotificationChannel;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
@ -29,11 +28,11 @@ import java.time.Duration;
import java.util.Map; import java.util.Map;
@Component @Component
@ConditionalOnProperty(value = "monitoring.notification_channels.slack.enabled", havingValue = "true") @ConditionalOnProperty(value = "monitoring.notifications.slack.enabled", havingValue = "true")
@Slf4j @Slf4j
public class SlackNotificationChannel implements NotificationChannel { public class SlackNotificationChannel implements NotificationChannel {
@Value("${monitoring.notification_channels.slack.webhook_url}") @Value("${monitoring.notifications.slack.webhook_url}")
private String webhookUrl; private String webhookUrl;
private RestTemplate restTemplate; private RestTemplate restTemplate;
@ -47,11 +46,7 @@ public class SlackNotificationChannel implements NotificationChannel {
} }
@Override @Override
public void sendNotification(Notification notification) { public void sendNotification(String message) {
sendNotification(notification.getText());
}
private void sendNotification(String message) {
restTemplate.postForObject(webhookUrl, Map.of("text", message), String.class); restTemplate.postForObject(webhookUrl, Map.of("text", message), String.class);
} }

96
monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java

@ -29,6 +29,8 @@ import org.thingsboard.monitoring.service.BaseHealthChecker;
import org.thingsboard.monitoring.util.ResourceUtils; import org.thingsboard.monitoring.util.ResourceUtils;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileType;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MBootstrapClientCredentials; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MBootstrapClientCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MDeviceCredentials;
@ -38,6 +40,9 @@ import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.data.DeviceData;
import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.security.DeviceCredentialsType;
@ -45,27 +50,19 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType;
@Slf4j @Slf4j
public abstract class TransportHealthChecker<C extends TransportMonitoringConfig> extends BaseHealthChecker<C, TransportMonitoringTarget> { public abstract class TransportHealthChecker<C extends TransportMonitoringConfig> extends BaseHealthChecker<C, TransportMonitoringTarget> {
private static final String DEFAULT_DEVICE_NAME = "[Monitoring] %s transport (%s)";
private static final String DEFAULT_PROFILE_NAME = "[Monitoring] %s";
public TransportHealthChecker(C config, TransportMonitoringTarget target) { public TransportHealthChecker(C config, TransportMonitoringTarget target) {
super(config, target); super(config, target);
} }
@Override @Override
protected void initialize(TbClient tbClient) { protected void initialize(TbClient tbClient) {
String deviceName = String.format(DEFAULT_DEVICE_NAME, config.getTransportType(), target.getBaseUrl()); Device device = getOrCreateDevice(tbClient);
Device device = tbClient.getTenantDevice(deviceName)
.orElseGet(() -> {
log.info("Creating new device '{}'", deviceName);
return createDevice(config.getTransportType(), deviceName, tbClient);
});
DeviceCredentials credentials = tbClient.getDeviceCredentialsByDeviceId(device.getId()) DeviceCredentials credentials = tbClient.getDeviceCredentialsByDeviceId(device.getId())
.orElseThrow(() -> new IllegalArgumentException("No credentials found for device " + device.getId())); .orElseThrow(() -> new IllegalArgumentException("No credentials found for device " + device.getId()));
DeviceConfig deviceConfig = new DeviceConfig(); DeviceConfig deviceConfig = new DeviceConfig();
deviceConfig.setId(device.getId().toString()); deviceConfig.setId(device.getId().toString());
deviceConfig.setName(deviceName); deviceConfig.setName(device.getName());
deviceConfig.setCredentials(credentials); deviceConfig.setCredentials(credentials);
target.setDevice(deviceConfig); target.setDevice(deviceConfig);
} }
@ -77,51 +74,43 @@ public abstract class TransportHealthChecker<C extends TransportMonitoringConfig
@Override @Override
protected Object getInfo() { protected Object getInfo() {
return new TransportInfo(getTransportType(), target.getBaseUrl()); return new TransportInfo(getTransportType(), target.getBaseUrl(), target.getQueue());
} }
@Override @Override
protected String getKey() { protected String getKey() {
return getTransportType().name().toLowerCase() + "Transport"; return getTransportType().name().toLowerCase() + (target.getQueue().equals("Main") ? "" : target.getQueue()) + "Transport";
} }
protected abstract TransportType getTransportType(); protected abstract TransportType getTransportType();
private Device createDevice(TransportType transportType, String name, TbClient tbClient) { private Device getOrCreateDevice(TbClient tbClient) {
Device device = new Device(); TransportType transportType = config.getTransportType();
device.setName(name); String deviceName = String.format("%s (%s) - %s", transportType.getName(), target.getQueue(), target.getBaseUrl());
Device device = tbClient.getTenantDevice(deviceName).orElse(null);
if (device != null) {
return device;
}
log.info("Creating new device '{}'", deviceName);
device = new Device();
device.setName(deviceName);
DeviceCredentials credentials = new DeviceCredentials(); DeviceCredentials credentials = new DeviceCredentials();
credentials.setCredentialsId(RandomStringUtils.randomAlphabetic(20)); credentials.setCredentialsId(RandomStringUtils.randomAlphabetic(20));
DeviceData deviceData = new DeviceData(); DeviceData deviceData = new DeviceData();
deviceData.setConfiguration(new DefaultDeviceConfiguration()); deviceData.setConfiguration(new DefaultDeviceConfiguration());
DeviceProfile deviceProfile = getOrCreateDeviceProfile(tbClient);
device.setType(deviceProfile.getName());
device.setDeviceProfileId(deviceProfile.getId());
if (transportType != TransportType.LWM2M) { if (transportType != TransportType.LWM2M) {
device.setType("default");
deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration());
credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN); credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN);
} else { } else {
tbClient.getResources(new PageLink(1, 0, "lwm2m monitoring")).getData()
.stream().findFirst()
.orElseGet(() -> {
TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class);
log.info("Creating LwM2M resource");
return tbClient.saveResource(newResource);
});
String profileName = String.format(DEFAULT_PROFILE_NAME, transportType);
DeviceProfile profile = tbClient.getDeviceProfiles(new PageLink(1, 0, profileName)).getData()
.stream().findFirst()
.orElseGet(() -> {
DeviceProfile newProfile = ResourceUtils.getResource("lwm2m/device_profile.json", DeviceProfile.class);
newProfile.setName(profileName);
log.info("Creating LwM2M device profile");
return tbClient.saveDeviceProfile(newProfile);
});
device.setType(profileName);
device.setDeviceProfileId(profile.getId());
deviceData.setTransportConfiguration(new Lwm2mDeviceTransportConfiguration()); deviceData.setTransportConfiguration(new Lwm2mDeviceTransportConfiguration());
credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS); credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS);
LwM2MDeviceCredentials lwm2mCreds = new LwM2MDeviceCredentials(); LwM2MDeviceCredentials lwm2mCreds = new LwM2MDeviceCredentials();
NoSecClientCredential client = new NoSecClientCredential(); NoSecClientCredential client = new NoSecClientCredential();
@ -133,7 +122,42 @@ public abstract class TransportHealthChecker<C extends TransportMonitoringConfig
lwm2mCreds.setBootstrap(bootstrap); lwm2mCreds.setBootstrap(bootstrap);
credentials.setCredentialsValue(JacksonUtil.toString(lwm2mCreds)); credentials.setCredentialsValue(JacksonUtil.toString(lwm2mCreds));
} }
return tbClient.saveDeviceWithCredentials(device, credentials).get(); return tbClient.saveDeviceWithCredentials(device, credentials).get();
} }
private DeviceProfile getOrCreateDeviceProfile(TbClient tbClient) {
TransportType transportType = config.getTransportType();
String profileName = String.format("%s (%s)", transportType.getName(), target.getQueue());
DeviceProfile deviceProfile = tbClient.getDeviceProfiles(new PageLink(1, 0, profileName)).getData()
.stream().findFirst().orElse(null);
if (deviceProfile != null) {
return deviceProfile;
}
log.info("Creating new device profile '{}'", profileName);
if (transportType != TransportType.LWM2M) {
deviceProfile = new DeviceProfile();
deviceProfile.setType(DeviceProfileType.DEFAULT);
deviceProfile.setTransportType(DeviceTransportType.DEFAULT);
DeviceProfileData profileData = new DeviceProfileData();
profileData.setConfiguration(new DefaultDeviceProfileConfiguration());
profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration());
deviceProfile.setProfileData(profileData);
} else {
tbClient.getResources(new PageLink(1, 0, "lwm2m monitoring")).getData()
.stream().findFirst()
.orElseGet(() -> {
TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class);
log.info("Creating LwM2M resource");
return tbClient.saveResource(newResource);
});
deviceProfile = ResourceUtils.getResource("lwm2m/device_profile.json", DeviceProfile.class);
}
deviceProfile.setName(profileName);
deviceProfile.setDefaultQueueName(target.getQueue());
return tbClient.saveDeviceProfile(deviceProfile);
}
} }

13
monitoring/src/main/resources/tb-monitoring.yml

@ -38,7 +38,7 @@ monitoring:
check_timeout_ms: '${CHECK_TIMEOUT_MS:5000}' check_timeout_ms: '${CHECK_TIMEOUT_MS:5000}'
# Failures threshold for notifying # Failures threshold for notifying
failures_threshold: '${FAILURES_THRESHOLD:2}' failures_threshold: '${FAILURES_THRESHOLD:1}'
# Notify after each REPEATED_FAILURE_NOTIFICATION subsequent failures, 0 to notify only once on first failure # Notify after each REPEATED_FAILURE_NOTIFICATION subsequent failures, 0 to notify only once on first failure
repeated_failure_notification: '${REPEATED_FAILURE_NOTIFICATION:4}' repeated_failure_notification: '${REPEATED_FAILURE_NOTIFICATION:4}'
@ -53,6 +53,8 @@ monitoring:
targets: targets:
# MQTT transport base url, tcp://DOMAIN:1883 by default # MQTT transport base url, tcp://DOMAIN:1883 by default
- base_url: '${MQTT_TRANSPORT_BASE_URL:tcp://${monitoring.domain}:1883}' - base_url: '${MQTT_TRANSPORT_BASE_URL:tcp://${monitoring.domain}:1883}'
# Queue to use for target device
queue: '${MQTT_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url # Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${MQTT_TRANSPORT_CHECK_DOMAIN_IPS:false}' check_domain_ips: '${MQTT_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# To add more targets, use following environment variables: # To add more targets, use following environment variables:
@ -66,6 +68,8 @@ monitoring:
targets: targets:
# CoAP transport base url, coap://DOMAIN by default # CoAP transport base url, coap://DOMAIN by default
- base_url: '${COAP_TRANSPORT_BASE_URL:coap://${monitoring.domain}}' - base_url: '${COAP_TRANSPORT_BASE_URL:coap://${monitoring.domain}}'
# Queue to use for target device
queue: '${COAP_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url # Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${COAP_TRANSPORT_CHECK_DOMAIN_IPS:false}' check_domain_ips: '${COAP_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# To add more targets, use following environment variables: # To add more targets, use following environment variables:
@ -79,6 +83,8 @@ monitoring:
targets: targets:
# HTTP transport base url, http://DOMAIN by default # HTTP transport base url, http://DOMAIN by default
- base_url: '${HTTP_TRANSPORT_BASE_URL:http://${monitoring.domain}}' - base_url: '${HTTP_TRANSPORT_BASE_URL:http://${monitoring.domain}}'
# Queue to use for target device
queue: '${HTTP_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url # Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${HTTP_TRANSPORT_CHECK_DOMAIN_IPS:false}' check_domain_ips: '${HTTP_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# To add more targets, use following environment variables: # To add more targets, use following environment variables:
@ -92,12 +98,15 @@ monitoring:
targets: targets:
# LwM2M transport base url, coap://DOMAIN:5685 by default # LwM2M transport base url, coap://DOMAIN:5685 by default
- base_url: '${LWM2M_TRANSPORT_BASE_URL:coap://${monitoring.domain}:5685}' - base_url: '${LWM2M_TRANSPORT_BASE_URL:coap://${monitoring.domain}:5685}'
# Queue to use for target device
queue: '${LWM2M_TRANSPORT_USED_QUEUE:Main}'
# Whether to monitor IPs associated with the domain from base url # Whether to monitor IPs associated with the domain from base url
check_domain_ips: '${LWM2M_TRANSPORT_CHECK_DOMAIN_IPS:false}' check_domain_ips: '${LWM2M_TRANSPORT_CHECK_DOMAIN_IPS:false}'
# To add more targets, use following environment variables: # To add more targets, use following environment variables:
# monitoring.transports.lwm2m.targets[1].base_url, monitoring.transports.lwm2m.targets[2].base_url, etc. # monitoring.transports.lwm2m.targets[1].base_url, monitoring.transports.lwm2m.targets[2].base_url, etc.
notification_channels: notifications:
message_prefix: '${NOTIFICATION_MESSAGE_PREFIX:}'
slack: slack:
# Enable notifying via Slack # Enable notifying via Slack
enabled: '${SLACK_NOTIFICATION_CHANNEL_ENABLED:false}' enabled: '${SLACK_NOTIFICATION_CHANNEL_ENABLED:false}'

2
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@ -107,6 +107,7 @@ public class ContainerTestSuite {
List<File> composeFiles = new ArrayList<>(Arrays.asList( List<File> composeFiles = new ArrayList<>(Arrays.asList(
new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.yml"),
new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.volumes.yml"),
new File(targetDir + "docker-compose.mosquitto.yml"),
new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")),
new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")),
new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"),
@ -162,6 +163,7 @@ public class ContainerTestSuite {
.withEnv(queueEnv) .withEnv(queueEnv)
.withEnv("LOAD_BALANCER_NAME", "") .withEnv("LOAD_BALANCER_NAME", "")
.withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
.withExposedService("broker", 1883)
.waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
.waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))
.waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT))

9
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java

@ -48,4 +48,13 @@ public class TestProperties {
} }
return System.getProperty("tb.wsUrl", "ws://localhost:8080"); return System.getProperty("tb.wsUrl", "ws://localhost:8080");
} }
public static String getMqttBrokerUrl() {
if (instance.isActive()) {
String host = instance.getTestContainer().getServiceHost("broker", 1883);
Integer port = instance.getTestContainer().getServicePort("broker", 1883);
return "tcp://" + host + ":" + port;
}
return System.getProperty("mqtt.broker", "tcp://localhost:1883");
}
} }

41
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java

@ -31,10 +31,12 @@ import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AlarmId;
import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.AssetProfileId;
@ -45,9 +47,11 @@ import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChain;
@ -527,4 +531,41 @@ public class TestRestClient {
.then() .then()
.statusCode(HTTP_OK); .statusCode(HTTP_OK);
} }
public PageData<EventInfo> getEvents(EntityId entityId, EventType eventType, TenantId tenantId, TimePageLink pageLink) {
Map<String, String> params = new HashMap<>();
params.put("entityType", entityId.getEntityType().name());
params.put("entityId", entityId.getId().toString());
params.put("eventType", eventType.name());
params.put("tenantId", tenantId.getId().toString());
addTimePageLinkToParam(params, pageLink);
return given().spec(requestSpec)
.get("/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&" + getTimeUrlParams(pageLink), params)
.then()
.statusCode(HTTP_OK)
.extract()
.as(new TypeRef<>() {});
}
private void addTimePageLinkToParam(Map<String, String> params, TimePageLink pageLink) {
this.addPageLinkToParam(params, pageLink);
if (pageLink.getStartTime() != null) {
params.put("startTime", String.valueOf(pageLink.getStartTime()));
}
if (pageLink.getEndTime() != null) {
params.put("endTime", String.valueOf(pageLink.getEndTime()));
}
}
private String getTimeUrlParams(TimePageLink pageLink) {
String urlParams = getUrlParams(pageLink);
if (pageLink.getStartTime() != null) {
urlParams += "&startTime={startTime}";
}
if (pageLink.getEndTime() != null) {
urlParams += "&endTime={endTime}";
}
return urlParams;
}
} }

204
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/rule/node/MqttNodeTest.java

@ -0,0 +1,204 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.msa.rule.node;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.eclipse.paho.client.mqttv3.IMqttMessageListener;
import org.eclipse.paho.client.mqttv3.MqttClient;
import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
import org.eclipse.paho.client.mqttv3.MqttMessage;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.rule.NodeConnectionInfo;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.msa.AbstractContainerTest;
import org.thingsboard.server.msa.DisableUIListeners;
import org.thingsboard.server.msa.TestProperties;
import org.thingsboard.server.msa.WsClient;
import org.thingsboard.server.msa.mapper.WsTelemetryResponse;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testng.Assert.fail;
import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype;
@DisableUIListeners
@Slf4j
public class MqttNodeTest extends AbstractContainerTest {
private static final String TOPIC = "tb/mqtt/device";
private Device device;
@BeforeMethod
public void setUp() {
testRestClient.login("tenant@thingsboard.org", "tenant");
device = testRestClient.postDevice("", defaultDevicePrototype("mqtt_"));
}
@AfterMethod
public void tearDown() {
testRestClient.deleteDeviceIfExists(device.getId());
}
@Test
public void telemetryUpload() throws Exception {
RuleChainId defaultRuleChainId = getDefaultRuleChainId();
createRootRuleChainWithTestNode("MqttRuleNodeTestMetadata.json", "org.thingsboard.rule.engine.mqtt.TbMqttNode", 2);
DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId());
WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS);
MqttMessageListener messageListener = new MqttMessageListener();
MqttClient responseClient = new MqttClient(TestProperties.getMqttBrokerUrl(), StringUtils.randomAlphanumeric(10), new MemoryPersistence());
responseClient.connect();
responseClient.subscribe(TOPIC, messageListener);
MqttClient mqttClient = new MqttClient("tcp://localhost:1883", StringUtils.randomAlphanumeric(10), new MemoryPersistence());
MqttConnectOptions mqttConnectOptions = new MqttConnectOptions();
mqttConnectOptions.setUserName(deviceCredentials.getCredentialsId());
mqttClient.connect(mqttConnectOptions);
mqttClient.publish("v1/devices/me/telemetry", new MqttMessage(createPayload().toString().getBytes()));
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
assertThat(actualLatestTelemetry.getData()).hasSize(4);
assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey"));
assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString());
assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1");
assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0));
assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73));
Awaitility
.await()
.alias("Get integration events")
.atMost(10, TimeUnit.SECONDS)
.until(() -> messageListener.getEvents().size() > 0);
BlockingQueue<MqttEvent> events = messageListener.getEvents();
JsonNode actual = JacksonUtil.toJsonNode(Objects.requireNonNull(events.poll()).message);
assertThat(actual.get("stringKey").asText()).isEqualTo("value1");
assertThat(actual.get("booleanKey").asBoolean()).isEqualTo(Boolean.TRUE);
assertThat(actual.get("doubleKey").asDouble()).isEqualTo(42.0);
assertThat(actual.get("longKey").asLong()).isEqualTo(73);
testRestClient.setRootRuleChain(defaultRuleChainId);
}
@Data
private class MqttMessageListener implements IMqttMessageListener {
private final BlockingQueue<MqttEvent> events;
private MqttMessageListener() {
events = new ArrayBlockingQueue<>(100);
}
@Override
public void messageArrived(String s, MqttMessage mqttMessage) {
log.info("MQTT message [{}], topic [{}]", mqttMessage.toString(), s);
events.add(new MqttEvent(s, mqttMessage.toString()));
}
public BlockingQueue<MqttEvent> getEvents() {
return events;
}
}
@Data
private class MqttEvent {
private final String topic;
private final String message;
}
private RuleChainId getDefaultRuleChainId() {
PageData<RuleChain> ruleChains = testRestClient.getRuleChains(new PageLink(40, 0));
Optional<RuleChain> defaultRuleChain = ruleChains.getData()
.stream()
.filter(RuleChain::isRoot)
.findFirst();
if (!defaultRuleChain.isPresent()) {
fail("Root rule chain wasn't found");
}
return defaultRuleChain.get().getId();
}
protected RuleChainId createRootRuleChainWithTestNode(String ruleChainMetadataFile, String ruleNodeType, int eventsCount) throws Exception {
RuleChain newRuleChain = new RuleChain();
newRuleChain.setName("testRuleChain");
RuleChain ruleChain = testRestClient.postRuleChain(newRuleChain);
JsonNode configuration = JacksonUtil.OBJECT_MAPPER.readTree(this.getClass().getClassLoader().getResourceAsStream(ruleChainMetadataFile));
RuleChainMetaData ruleChainMetaData = new RuleChainMetaData();
ruleChainMetaData.setRuleChainId(ruleChain.getId());
ruleChainMetaData.setFirstNodeIndex(configuration.get("firstNodeIndex").asInt());
ruleChainMetaData.setNodes(Arrays.asList(JacksonUtil.OBJECT_MAPPER.treeToValue(configuration.get("nodes"), RuleNode[].class)));
ruleChainMetaData.setConnections(Arrays.asList(JacksonUtil.OBJECT_MAPPER.treeToValue(configuration.get("connections"), NodeConnectionInfo[].class)));
ruleChainMetaData = testRestClient.postRuleChainMetadata(ruleChainMetaData);
testRestClient.setRootRuleChain(ruleChain.getId());
RuleNode node = ruleChainMetaData.getNodes().stream().filter(ruleNode -> ruleNode.getType().equals(ruleNodeType)).findFirst().get();
Awaitility
.await()
.alias("Get events from rule chain")
.atMost(10, TimeUnit.SECONDS)
.until(() -> {
PageData<EventInfo> events = testRestClient.getEvents(node.getId(), EventType.LC_EVENT, ruleChain.getTenantId(), new TimePageLink(1024));
List<EventInfo> eventInfos = events.getData().stream().filter(eventInfo ->
"STARTED".equals(eventInfo.getBody().get("event").asText()) &&
"true".equals(eventInfo.getBody().get("success").asText()))
.collect(Collectors.toList());
return eventInfos.size() == eventsCount;
});
return ruleChain.getId();
}
}

79
msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json

@ -0,0 +1,79 @@
{
"firstNodeIndex": 2,
"nodes": [
{
"additionalInfo": {
"description": "",
"layoutX": 626,
"layoutY": 152
},
"type": "org.thingsboard.rule.engine.mqtt.TbMqttNode",
"name": "test mqtt",
"debugMode": true,
"singletonMode": true,
"queueName": "HighPriority",
"configurationVersion": 0,
"configuration": {
"topicPattern": "tb/mqtt/device",
"host": "broker",
"port": 1883,
"connectTimeoutSec": 10,
"clientId": null,
"cleanSession": true,
"retainedMessage": false,
"ssl": false,
"credentials": {
"type": "anonymous"
}
},
"externalId": null
},
{
"additionalInfo": {
"description": "",
"layoutX": 949,
"layoutY": 153
},
"type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode",
"name": "save timeseries",
"debugMode": true,
"singletonMode": false,
"configurationVersion": 0,
"configuration": {
"defaultTTL": 0,
"skipLatestPersistence": false,
"useServerTs": false
},
"externalId": null
},
{
"additionalInfo": {
"description": "",
"layoutX": 305,
"layoutY": 151
},
"type": "org.thingsboard.rule.engine.filter.TbMsgTypeSwitchNode",
"name": "switch",
"debugMode": false,
"singletonMode": false,
"configurationVersion": 0,
"configuration": {
"version": 0
},
"externalId": null
}
],
"connections": [
{
"fromIndex": 0,
"toIndex": 1,
"type": "Success"
},
{
"fromIndex": 2,
"toIndex": 0,
"type": "Post telemetry"
}
],
"ruleChainConnections": null
}

25
msa/black-box-tests/src/test/resources/docker-compose.mosquitto.yml

@ -0,0 +1,25 @@
#
# Copyright © 2016-2024 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '3.0'
services:
broker:
image: eclipse-mosquitto
volumes:
- ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf
ports:
- "1883"
restart: always

18
msa/black-box-tests/src/test/resources/mosquitto/mosquitto.conf

@ -0,0 +1,18 @@
#
# Copyright © 2016-2024 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
listener 1883
allow_anonymous true

179
rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java

@ -29,6 +29,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.http.client.support.HttpRequestWrapper; import org.springframework.http.client.support.HttpRequestWrapper;
import org.springframework.util.CollectionUtils;
import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap; import org.springframework.util.MultiValueMap;
import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.HttpClientErrorException;
@ -160,6 +161,7 @@ import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.widget.DeprecatedFilter;
import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.common.data.widget.WidgetTypeInfo;
@ -2713,15 +2715,33 @@ public class RestClient implements Closeable {
return restTemplate.postForEntity(baseURL + "/api/widgetsBundle", widgetsBundle, WidgetsBundle.class).getBody(); return restTemplate.postForEntity(baseURL + "/api/widgetsBundle", widgetsBundle, WidgetsBundle.class).getBody();
} }
public void updateWidgetsBundleWidgetTypes(WidgetsBundleId widgetsBundleId, List<WidgetTypeId> widgetTypeIds) {
var httpEntity = new HttpEntity<>(widgetTypeIds.stream()
.map(widgetTypeId -> widgetTypeId.getId().toString())
.collect(Collectors.toList()));
restTemplate.exchange(baseURL + "/api/widgetsBundle/{widgetsBundleId}/widgetTypes",
HttpMethod.POST, httpEntity, Void.class, widgetsBundleId.getId());
}
public void updateWidgetsBundleWidgetFqns(WidgetsBundleId widgetsBundleId, List<String> widgetTypeFqns) {
restTemplate.exchange(baseURL + "/api/widgetsBundle/{widgetsBundleId}/widgetTypeFqns",
HttpMethod.POST, new HttpEntity<>(widgetTypeFqns), Void.class, widgetsBundleId.getId());
}
public void deleteWidgetsBundle(WidgetsBundleId widgetsBundleId) { public void deleteWidgetsBundle(WidgetsBundleId widgetsBundleId) {
restTemplate.delete(baseURL + "/api/widgetsBundle/{widgetsBundleId}", widgetsBundleId.getId()); restTemplate.delete(baseURL + "/api/widgetsBundle/{widgetsBundleId}", widgetsBundleId.getId());
} }
public PageData<WidgetsBundle> getWidgetsBundles(PageLink pageLink) { public PageData<WidgetsBundle> getWidgetsBundles(PageLink pageLink) {
return getWidgetsBundles(pageLink, null, null);
}
public PageData<WidgetsBundle> getWidgetsBundles(PageLink pageLink, Boolean tenantOnly, Boolean fullSearch) {
Map<String, String> params = new HashMap<>(); Map<String, String> params = new HashMap<>();
addPageLinkToParam(params, pageLink); addPageLinkToParam(params, pageLink);
addTenantOnlyAndFullSearchToParams(tenantOnly, fullSearch, params);
return restTemplate.exchange( return restTemplate.exchange(
baseURL + "/api/widgetsBundles?" + getUrlParams(pageLink), baseURL + "/api/widgetsBundles?" + getUrlParams(pageLink) + getTenantOnlyAndFullSearchUrlParams(tenantOnly, fullSearch),
HttpMethod.GET, HttpMethod.GET,
HttpEntity.EMPTY, HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<WidgetsBundle>>() { new ParameterizedTypeReference<PageData<WidgetsBundle>>() {
@ -2751,14 +2771,54 @@ public class RestClient implements Closeable {
} }
} }
public Optional<WidgetTypeInfo> getWidgetTypeInfoById(WidgetTypeId widgetTypeId) {
try {
ResponseEntity<WidgetTypeInfo> widgetTypeInfo =
restTemplate.getForEntity(baseURL + "/api/widgetTypeInfo/{widgetTypeId}", WidgetTypeInfo.class, widgetTypeId.getId());
return Optional.ofNullable(widgetTypeInfo.getBody());
} catch (HttpClientErrorException exception) {
if (exception.getStatusCode() == HttpStatus.NOT_FOUND) {
return Optional.empty();
}
throw exception;
}
}
public WidgetTypeDetails saveWidgetType(WidgetTypeDetails widgetTypeDetails) { public WidgetTypeDetails saveWidgetType(WidgetTypeDetails widgetTypeDetails) {
return restTemplate.postForEntity(baseURL + "/api/widgetType", widgetTypeDetails, WidgetTypeDetails.class).getBody(); return saveWidgetType(widgetTypeDetails, null);
}
public WidgetTypeDetails saveWidgetType(WidgetTypeDetails widgetTypeDetails, Boolean updateExistingByFqn) {
if (updateExistingByFqn == null) {
return restTemplate.postForEntity(baseURL + "/api/widgetType", widgetTypeDetails, WidgetTypeDetails.class).getBody();
}
return restTemplate.postForEntity(baseURL + "/api/widgetType?updateExistingByFqn={updateExistingByFqn}", widgetTypeDetails, WidgetTypeDetails.class, updateExistingByFqn).getBody();
} }
public void deleteWidgetType(WidgetTypeId widgetTypeId) { public void deleteWidgetType(WidgetTypeId widgetTypeId) {
restTemplate.delete(baseURL + "/api/widgetType/{widgetTypeId}", widgetTypeId.getId()); restTemplate.delete(baseURL + "/api/widgetType/{widgetTypeId}", widgetTypeId.getId());
} }
public PageData<WidgetTypeInfo> getWidgetTypes(PageLink pageLink) {
return getWidgetTypes(pageLink, null, null, null, null);
}
public PageData<WidgetTypeInfo> getWidgetTypes(PageLink pageLink, Boolean tenantOnly, Boolean fullSearch,
DeprecatedFilter deprecatedFilter, List<String> widgetTypeList) {
Map<String, String> params = new HashMap<>();
addPageLinkToParam(params, pageLink);
addWidgetInfoFiltersToParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList, params);
return restTemplate.exchange(
baseURL + "/api/widgetTypes?" + getUrlParams(pageLink) +
getWidgetTypeInfoPageRequestUrlParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList),
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<WidgetTypeInfo>>() {
},
params).getBody();
}
@Deprecated // current name in the controller: getBundleWidgetTypesByBundleAlias
public List<WidgetType> getBundleWidgetTypes(boolean isSystem, String bundleAlias) { public List<WidgetType> getBundleWidgetTypes(boolean isSystem, String bundleAlias) {
return restTemplate.exchange( return restTemplate.exchange(
baseURL + "/api/widgetTypes?isSystem={isSystem}&bundleAlias={bundleAlias}", baseURL + "/api/widgetTypes?isSystem={isSystem}&bundleAlias={bundleAlias}",
@ -2770,6 +2830,17 @@ public class RestClient implements Closeable {
bundleAlias).getBody(); bundleAlias).getBody();
} }
public List<WidgetType> getBundleWidgetTypes(WidgetsBundleId widgetsBundleId) {
return restTemplate.exchange(
baseURL + "/api/widgetTypes?widgetsBundleId={widgetsBundleId}",
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<List<WidgetType>>() {
},
widgetsBundleId.getId()).getBody();
}
@Deprecated // current name in the controller: getBundleWidgetTypesDetailsByBundleAlias
public List<WidgetTypeDetails> getBundleWidgetTypesDetails(boolean isSystem, String bundleAlias) { public List<WidgetTypeDetails> getBundleWidgetTypesDetails(boolean isSystem, String bundleAlias) {
return restTemplate.exchange( return restTemplate.exchange(
baseURL + "/api/widgetTypesDetails?isSystem={isSystem}&bundleAlias={bundleAlias}", baseURL + "/api/widgetTypesDetails?isSystem={isSystem}&bundleAlias={bundleAlias}",
@ -2781,6 +2852,28 @@ public class RestClient implements Closeable {
bundleAlias).getBody(); bundleAlias).getBody();
} }
public List<WidgetTypeDetails> getBundleWidgetTypesDetails(WidgetsBundleId widgetsBundleId, boolean inlineImages) {
return restTemplate.exchange(
baseURL + "/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}&inlineImages={inlineImages}",
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<List<WidgetTypeDetails>>() {
},
widgetsBundleId.getId(),
inlineImages).getBody();
}
public List<String> getBundleWidgetTypeFqns(WidgetsBundleId widgetsBundleId) {
return restTemplate.exchange(
baseURL + "/api/widgetTypeFqns?widgetsBundleId={widgetsBundleId}",
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<List<String>>() {
},
widgetsBundleId.getId()).getBody();
}
@Deprecated // current name in the controller: getBundleWidgetTypesInfosByBundleAlias
public List<WidgetTypeInfo> getBundleWidgetTypesInfos(boolean isSystem, String bundleAlias) { public List<WidgetTypeInfo> getBundleWidgetTypesInfos(boolean isSystem, String bundleAlias) {
return restTemplate.exchange( return restTemplate.exchange(
baseURL + "/api/widgetTypesInfos?isSystem={isSystem}&bundleAlias={bundleAlias}", baseURL + "/api/widgetTypesInfos?isSystem={isSystem}&bundleAlias={bundleAlias}",
@ -2792,6 +2885,28 @@ public class RestClient implements Closeable {
bundleAlias).getBody(); bundleAlias).getBody();
} }
public PageData<WidgetTypeInfo> getBundleWidgetTypesInfos(WidgetsBundleId widgetsBundleId, PageLink pageLink) {
return getBundleWidgetTypesInfos(widgetsBundleId, pageLink, null, null, null, null);
}
public PageData<WidgetTypeInfo> getBundleWidgetTypesInfos(WidgetsBundleId widgetsBundleId, PageLink pageLink,
Boolean tenantOnly, Boolean fullSearch,
DeprecatedFilter deprecatedFilter, List<String> widgetTypeList) {
Map<String, String> params = new HashMap<>();
params.put("widgetsBundleId", widgetsBundleId.getId().toString());
addPageLinkToParam(params, pageLink);
addWidgetInfoFiltersToParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList, params);
return restTemplate.exchange(
baseURL + "/api/widgetTypesInfos?widgetsBundleId={widgetsBundleId}&" + getUrlParams(pageLink) +
getWidgetTypeInfoPageRequestUrlParams(tenantOnly, fullSearch, deprecatedFilter, widgetTypeList),
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<WidgetTypeInfo>>() {
},
params).getBody();
}
@Deprecated // current name in the controller: getWidgetTypeByBundleAliasAndTypeAlias
public Optional<WidgetType> getWidgetType(boolean isSystem, String bundleAlias, String alias) { public Optional<WidgetType> getWidgetType(boolean isSystem, String bundleAlias, String alias) {
try { try {
ResponseEntity<WidgetType> widgetType = ResponseEntity<WidgetType> widgetType =
@ -2811,6 +2926,22 @@ public class RestClient implements Closeable {
} }
} }
public Optional<WidgetType> getWidgetType(String fqn) {
try {
ResponseEntity<WidgetType> widgetType =
restTemplate.getForEntity(
baseURL + "/api/widgetType?fqn={fqn}",
WidgetType.class,
fqn);
return Optional.ofNullable(widgetType.getBody());
} catch (HttpClientErrorException exception) {
if (exception.getStatusCode() == HttpStatus.NOT_FOUND) {
return Optional.empty();
}
throw exception;
}
}
public Boolean isEdgesSupportEnabled() { public Boolean isEdgesSupportEnabled() {
return restTemplate.getForEntity(baseURL + "/api/edges/enabled", Boolean.class).getBody(); return restTemplate.getForEntity(baseURL + "/api/edges/enabled", Boolean.class).getBody();
} }
@ -3645,6 +3776,30 @@ public class RestClient implements Closeable {
return urlParams; return urlParams;
} }
private String getWidgetTypeInfoPageRequestUrlParams(Boolean tenantOnly, Boolean fullSearch,
DeprecatedFilter deprecatedFilter,
List<String> widgetTypeList) {
String urlParams = getTenantOnlyAndFullSearchUrlParams(tenantOnly, fullSearch);
if (deprecatedFilter != null) {
urlParams += "&deprecatedFilter={deprecatedFilter}";
}
if (!CollectionUtils.isEmpty(widgetTypeList)) {
urlParams += "&widgetTypeList={widgetTypeList}";
}
return urlParams;
}
private String getTenantOnlyAndFullSearchUrlParams(Boolean tenantOnly, Boolean fullSearch) {
String urlParams = "";
if (tenantOnly != null) {
urlParams = "&tenantOnly={tenantOnly}";
}
if (fullSearch != null) {
urlParams += "&fullSearch={fullSearch}";
}
return urlParams;
}
private void addTimePageLinkToParam(Map<String, String> params, TimePageLink pageLink) { private void addTimePageLinkToParam(Map<String, String> params, TimePageLink pageLink) {
this.addPageLinkToParam(params, pageLink); this.addPageLinkToParam(params, pageLink);
if (pageLink.getStartTime() != null) { if (pageLink.getStartTime() != null) {
@ -3667,6 +3822,26 @@ public class RestClient implements Closeable {
} }
} }
private void addWidgetInfoFiltersToParams(Boolean tenantOnly, Boolean fullSearch, DeprecatedFilter deprecatedFilter,
List<String> widgetTypeList, Map<String, String> params) {
addTenantOnlyAndFullSearchToParams(tenantOnly, fullSearch, params);
if (deprecatedFilter != null) {
params.put("deprecatedFilter", deprecatedFilter.name());
}
if (!CollectionUtils.isEmpty(widgetTypeList)) {
params.put("widgetTypeList", listToString(widgetTypeList));
}
}
private void addTenantOnlyAndFullSearchToParams(Boolean tenantOnly, Boolean fullSearch, Map<String, String> params) {
if (tenantOnly != null) {
params.put("tenantOnly", tenantOnly.toString());
}
if (fullSearch != null) {
params.put("fullSearch", fullSearch.toString());
}
}
private String listToString(List<String> list) { private String listToString(List<String> list) {
return String.join(",", list); return String.join(",", list);
} }

2
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleNode.java

@ -42,6 +42,8 @@ public @interface RuleNode {
ComponentClusteringMode clusteringMode() default ComponentClusteringMode.ENABLED; ComponentClusteringMode clusteringMode() default ComponentClusteringMode.ENABLED;
boolean hasQueueName() default false;
boolean inEnabled() default true; boolean inEnabled() default true;
boolean outEnabled() default true; boolean outEnabled() default true;

2
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java

@ -256,6 +256,8 @@ public interface TbContext {
String getRuleChainName(); String getRuleChainName();
String getQueueName();
TenantId getTenantId(); TenantId getTenantId();
AttributesService getAttributesService(); AttributesService getAttributesService();

1
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/util/TbNodeUtils.java

@ -36,7 +36,6 @@ import java.util.stream.Collectors;
*/ */
public class TbNodeUtils { public class TbNodeUtils {
private static final Pattern DATA_PATTERN = Pattern.compile("(\\$\\[)(.*?)(])"); private static final Pattern DATA_PATTERN = Pattern.compile("(\\$\\[)(.*?)(])");
public static <T> T convert(TbNodeConfiguration configuration, Class<T> clazz) throws TbNodeException { public static <T> T convert(TbNodeConfiguration configuration, Class<T> clazz) throws TbNodeException {

30
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java

@ -15,6 +15,8 @@
*/ */
package org.thingsboard.rule.engine.debug; package org.thingsboard.rule.engine.debug;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
@ -35,6 +37,7 @@ import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.msg.TbNodeConnectionType; import org.thingsboard.server.common.data.msg.TbNodeConnectionType;
import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.data.script.ScriptLanguage; import org.thingsboard.server.common.data.script.ScriptLanguage;
import org.thingsboard.server.common.data.util.TbPair;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg;
@ -44,12 +47,15 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import static org.thingsboard.common.util.DonAsynchron.withCallback; import static org.thingsboard.common.util.DonAsynchron.withCallback;
import static org.thingsboard.server.common.data.DataConstants.QUEUE_NAME;
@Slf4j @Slf4j
@RuleNode( @RuleNode(
type = ComponentType.ACTION, type = ComponentType.ACTION,
name = "generator", name = "generator",
configClazz = TbMsgGeneratorNodeConfiguration.class, configClazz = TbMsgGeneratorNodeConfiguration.class,
version = 1,
hasQueueName = true,
nodeDescription = "Periodically generates messages", nodeDescription = "Periodically generates messages",
nodeDetails = "Generates messages with configurable period. Javascript function used for message generation.", nodeDetails = "Generates messages with configurable period. Javascript function used for message generation.",
inEnabled = false, inEnabled = false,
@ -69,6 +75,7 @@ public class TbMsgGeneratorNode implements TbNode {
private UUID nextTickId; private UUID nextTickId;
private TbMsg prevMsg; private TbMsg prevMsg;
private final AtomicBoolean initialized = new AtomicBoolean(false); private final AtomicBoolean initialized = new AtomicBoolean(false);
private String queueName;
@Override @Override
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
@ -76,6 +83,7 @@ public class TbMsgGeneratorNode implements TbNode {
this.config = TbNodeUtils.convert(configuration, TbMsgGeneratorNodeConfiguration.class); this.config = TbNodeUtils.convert(configuration, TbMsgGeneratorNodeConfiguration.class);
this.delay = TimeUnit.SECONDS.toMillis(config.getPeriodInSeconds()); this.delay = TimeUnit.SECONDS.toMillis(config.getPeriodInSeconds());
this.currentMsgCount = 0; this.currentMsgCount = 0;
this.queueName = ctx.getQueueName();
if (!StringUtils.isEmpty(config.getOriginatorId())) { if (!StringUtils.isEmpty(config.getOriginatorId())) {
originatorId = EntityIdFactory.getByTypeAndUuid(config.getOriginatorType(), config.getOriginatorId()); originatorId = EntityIdFactory.getByTypeAndUuid(config.getOriginatorType(), config.getOriginatorId());
ctx.checkTenantEntity(originatorId); ctx.checkTenantEntity(originatorId);
@ -137,7 +145,7 @@ public class TbMsgGeneratorNode implements TbNode {
} }
lastScheduledTs = lastScheduledTs + delay; lastScheduledTs = lastScheduledTs + delay;
long curDelay = Math.max(0L, (lastScheduledTs - curTs)); long curDelay = Math.max(0L, (lastScheduledTs - curTs));
TbMsg tickMsg = ctx.newMsg(config.getQueueName(), TbMsgType.GENERATOR_NODE_SELF_MSG, ctx.getSelfId(), TbMsg tickMsg = ctx.newMsg(queueName, TbMsgType.GENERATOR_NODE_SELF_MSG, ctx.getSelfId(),
getCustomerIdFromMsg(msg), TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING); getCustomerIdFromMsg(msg), TbMsgMetaData.EMPTY, TbMsg.EMPTY_STRING);
nextTickId = tickMsg.getId(); nextTickId = tickMsg.getId();
ctx.tellSelf(tickMsg, curDelay); ctx.tellSelf(tickMsg, curDelay);
@ -146,14 +154,14 @@ public class TbMsgGeneratorNode implements TbNode {
private ListenableFuture<TbMsg> generate(TbContext ctx, TbMsg msg) { private ListenableFuture<TbMsg> generate(TbContext ctx, TbMsg msg) {
log.trace("generate, config {}", config); log.trace("generate, config {}", config);
if (prevMsg == null) { if (prevMsg == null) {
prevMsg = ctx.newMsg(config.getQueueName(), TbMsg.EMPTY_STRING, originatorId, msg.getCustomerId(), TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT); prevMsg = ctx.newMsg(queueName, TbMsg.EMPTY_STRING, originatorId, msg.getCustomerId(), TbMsgMetaData.EMPTY, TbMsg.EMPTY_JSON_OBJECT);
} }
if (initialized.get()) { if (initialized.get()) {
ctx.logJsEvalRequest(); ctx.logJsEvalRequest();
return Futures.transformAsync(scriptEngine.executeGenerateAsync(prevMsg), generated -> { return Futures.transformAsync(scriptEngine.executeGenerateAsync(prevMsg), generated -> {
log.trace("generate process response, generated {}, config {}", generated, config); log.trace("generate process response, generated {}, config {}", generated, config);
ctx.logJsEvalResponse(); ctx.logJsEvalResponse();
prevMsg = ctx.newMsg(config.getQueueName(), generated.getType(), originatorId, msg.getCustomerId(), generated.getMetaData(), generated.getData()); prevMsg = ctx.newMsg(queueName, generated.getType(), originatorId, msg.getCustomerId(), generated.getMetaData(), generated.getData());
return Futures.immediateFuture(prevMsg); return Futures.immediateFuture(prevMsg);
}, MoreExecutors.directExecutor()); //usually it runs on js-executor-remote-callback thread pool }, MoreExecutors.directExecutor()); //usually it runs on js-executor-remote-callback thread pool
} }
@ -174,4 +182,20 @@ public class TbMsgGeneratorNode implements TbNode {
scriptEngine = null; scriptEngine = null;
} }
} }
@Override
public TbPair<Boolean, JsonNode> upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException {
boolean hasChanges = false;
switch (fromVersion) {
case 0:
if (oldConfiguration.has(QUEUE_NAME)) {
hasChanges = true;
((ObjectNode) oldConfiguration).remove(QUEUE_NAME);
}
break;
default:
break;
}
return new TbPair<>(hasChanges, oldConfiguration);
}
} }

1
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNodeConfiguration.java

@ -36,7 +36,6 @@ public class TbMsgGeneratorNodeConfiguration implements NodeConfiguration<TbMsgG
private ScriptLanguage scriptLang; private ScriptLanguage scriptLang;
private String jsScript; private String jsScript;
private String tbelScript; private String tbelScript;
private String queueName;
@Override @Override
public TbMsgGeneratorNodeConfiguration defaultConfiguration() { public TbMsgGeneratorNodeConfiguration defaultConfiguration() {

27
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java

@ -15,6 +15,7 @@
*/ */
package org.thingsboard.rule.engine.deduplication; package org.thingsboard.rule.engine.deduplication;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -43,10 +44,14 @@ import java.util.Optional;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.thingsboard.server.common.data.DataConstants.QUEUE_NAME;
@RuleNode( @RuleNode(
type = ComponentType.TRANSFORMATION, type = ComponentType.TRANSFORMATION,
name = "deduplication", name = "deduplication",
configClazz = TbMsgDeduplicationNodeConfiguration.class, configClazz = TbMsgDeduplicationNodeConfiguration.class,
version = 1,
hasQueueName = true,
nodeDescription = "Deduplicate messages within the same originator entity for a configurable period " + nodeDescription = "Deduplicate messages within the same originator entity for a configurable period " +
"based on a specified deduplication strategy.", "based on a specified deduplication strategy.",
nodeDetails = "Deduplication strategies: <ul><li><strong>FIRST</strong> - return first message that arrived during deduplication period.</li>" + nodeDetails = "Deduplication strategies: <ul><li><strong>FIRST</strong> - return first message that arrived during deduplication period.</li>" +
@ -66,6 +71,7 @@ public class TbMsgDeduplicationNode implements TbNode {
private final Map<EntityId, DeduplicationData> deduplicationMap; private final Map<EntityId, DeduplicationData> deduplicationMap;
private long deduplicationInterval; private long deduplicationInterval;
private String queueName;
public TbMsgDeduplicationNode() { public TbMsgDeduplicationNode() {
this.deduplicationMap = new HashMap<>(); this.deduplicationMap = new HashMap<>();
@ -75,6 +81,7 @@ public class TbMsgDeduplicationNode implements TbNode {
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
this.config = TbNodeUtils.convert(configuration, TbMsgDeduplicationNodeConfiguration.class); this.config = TbNodeUtils.convert(configuration, TbMsgDeduplicationNodeConfiguration.class);
this.deduplicationInterval = TimeUnit.SECONDS.toMillis(config.getInterval()); this.deduplicationInterval = TimeUnit.SECONDS.toMillis(config.getInterval());
this.queueName = ctx.getQueueName();
} }
@Override @Override
@ -91,6 +98,22 @@ public class TbMsgDeduplicationNode implements TbNode {
deduplicationMap.clear(); deduplicationMap.clear();
} }
@Override
public TbPair<Boolean, JsonNode> upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException {
boolean hasChanges = false;
switch (fromVersion) {
case 0:
if (oldConfiguration.has(QUEUE_NAME)) {
hasChanges = true;
((ObjectNode) oldConfiguration).remove(QUEUE_NAME);
}
break;
default:
break;
}
return new TbPair<>(hasChanges, oldConfiguration);
}
private void processOnRegularMsg(TbContext ctx, TbMsg msg) { private void processOnRegularMsg(TbContext ctx, TbMsg msg) {
EntityId id = msg.getOriginator(); EntityId id = msg.getOriginator();
DeduplicationData deduplicationMsgs = deduplicationMap.computeIfAbsent(id, k -> new DeduplicationData()); DeduplicationData deduplicationMsgs = deduplicationMap.computeIfAbsent(id, k -> new DeduplicationData());
@ -132,7 +155,7 @@ public class TbMsgDeduplicationNode implements TbNode {
} }
} }
deduplicationResults.add(TbMsg.newMsg( deduplicationResults.add(TbMsg.newMsg(
config.getQueueName(), queueName,
config.getOutMsgType(), config.getOutMsgType(),
deduplicationId, deduplicationId,
getMetadata(), getMetadata(),
@ -154,7 +177,7 @@ public class TbMsgDeduplicationNode implements TbNode {
} }
if (resultMsg != null) { if (resultMsg != null) {
deduplicationResults.add(TbMsg.newMsg( deduplicationResults.add(TbMsg.newMsg(
resultMsg.getQueueName(), queueName != null ? queueName : resultMsg.getQueueName(),
resultMsg.getType(), resultMsg.getType(),
resultMsg.getOriginator(), resultMsg.getOriginator(),
resultMsg.getCustomerId(), resultMsg.getCustomerId(),

1
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNodeConfiguration.java

@ -26,7 +26,6 @@ public class TbMsgDeduplicationNodeConfiguration implements NodeConfiguration<Tb
// only for DeduplicationStrategy.ALL: // only for DeduplicationStrategy.ALL:
private String outMsgType; private String outMsgType;
private String queueName;
// Advanced settings: // Advanced settings:
private int maxPendingMsgs; private int maxPendingMsgs;

31
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/flow/TbCheckpointNode.java

@ -15,7 +15,10 @@
*/ */
package org.thingsboard.rule.engine.flow; package org.thingsboard.rule.engine.flow;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.rule.engine.api.EmptyNodeConfiguration;
import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.RuleNode;
import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNode;
@ -24,17 +27,22 @@ import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.server.common.data.msg.TbNodeConnectionType; import org.thingsboard.server.common.data.msg.TbNodeConnectionType;
import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.rule.engine.api.util.TbNodeUtils;
import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.data.util.TbPair;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import static org.thingsboard.server.common.data.DataConstants.QUEUE_NAME;
@Slf4j @Slf4j
@RuleNode( @RuleNode(
type = ComponentType.FLOW, type = ComponentType.FLOW,
name = "checkpoint", name = "checkpoint",
configClazz = TbCheckpointNodeConfiguration.class, configClazz = EmptyNodeConfiguration.class,
version = 1,
hasQueueName = true,
nodeDescription = "transfers the message to another queue", nodeDescription = "transfers the message to another queue",
nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.", nodeDetails = "After successful transfer incoming message is automatically acknowledged. Queue name is configurable.",
uiResources = {"static/rulenode/rulenode-core-config.js"}, uiResources = {"static/rulenode/rulenode-core-config.js"},
configDirective = "tbActionNodeCheckPointConfig" configDirective = "tbNodeEmptyConfig"
) )
public class TbCheckpointNode implements TbNode { public class TbCheckpointNode implements TbNode {
@ -42,8 +50,7 @@ public class TbCheckpointNode implements TbNode {
@Override @Override
public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
TbCheckpointNodeConfiguration config = TbNodeUtils.convert(configuration, TbCheckpointNodeConfiguration.class); this.queueName = ctx.getQueueName();
this.queueName = config.getQueueName();
} }
@Override @Override
@ -51,4 +58,20 @@ public class TbCheckpointNode implements TbNode {
ctx.enqueueForTellNext(msg, queueName, TbNodeConnectionType.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error)); ctx.enqueueForTellNext(msg, queueName, TbNodeConnectionType.SUCCESS, () -> ctx.ack(msg), error -> ctx.tellFailure(msg, error));
} }
@Override
public TbPair<Boolean, JsonNode> upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException {
boolean hasChanges = false;
switch (fromVersion) {
case 0:
if (oldConfiguration.has(QUEUE_NAME)) {
hasChanges = true;
((ObjectNode) oldConfiguration).remove(QUEUE_NAME);
}
break;
default:
break;
}
return new TbPair<>(hasChanges, oldConfiguration);
}
} }

50
rule-engine/rule-engine-components/src/main/resources/public/static/rulenode/rulenode-core-config.js

File diff suppressed because one or more lines are too long

52
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/AbstractRuleNodeUpgradeTest.java

@ -0,0 +1,52 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.api.TbNode;
import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.server.common.data.util.TbPair;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.BDDMockito.willCallRealMethod;
public abstract class AbstractRuleNodeUpgradeTest {
protected abstract TbNode getTestNode();
@ParameterizedTest
@MethodSource
public void givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig(int givenVersion, String givenConfigStr, boolean hasChanges, String expectedConfigStr) throws TbNodeException {
// GIVEN
willCallRealMethod().given(getTestNode()).upgrade(anyInt(), any());
JsonNode givenConfig = JacksonUtil.toJsonNode(givenConfigStr);
JsonNode expectedConfig = JacksonUtil.toJsonNode(expectedConfigStr);
// WHEN
TbPair<Boolean, JsonNode> upgradeResult = getTestNode().upgrade(givenVersion, givenConfig);
// THEN
assertThat(upgradeResult.getFirst()).isEqualTo(hasChanges);
ObjectNode upgradedConfig = (ObjectNode) upgradeResult.getSecond();
assertThat(upgradedConfig).isEqualTo(expectedConfig);
}
}

53
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNodeTest.java

@ -0,0 +1,53 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.debug;
import org.junit.jupiter.params.provider.Arguments;
import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest;
import org.thingsboard.rule.engine.api.TbNode;
import java.util.stream.Stream;
import static org.mockito.Mockito.spy;
public class TbMsgGeneratorNodeTest extends AbstractRuleNodeUpgradeTest {
// Rule nodes upgrade
private static Stream<Arguments> givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() {
return Stream.of(
// default config for version 0
Arguments.of(0,
"{\"msgCount\":0,\"periodInSeconds\":1,\"originatorId\":null,\"originatorType\":null, \"queueName\":null, \"scriptLang\":\"TBEL\",\"jsScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\",\"tbelScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\"}",
true,
"{\"msgCount\":0,\"periodInSeconds\":1,\"originatorId\":null,\"originatorType\":null, \"scriptLang\":\"TBEL\",\"jsScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\",\"tbelScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\"}"),
// default config for version 0 with queueName
Arguments.of(0,
"{\"msgCount\":0,\"periodInSeconds\":1,\"originatorId\":null,\"originatorType\":null, \"queueName\":\"Main\", \"scriptLang\":\"TBEL\",\"jsScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\",\"tbelScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\"}",
true,
"{\"msgCount\":0,\"periodInSeconds\":1,\"originatorId\":null,\"originatorType\":null, \"scriptLang\":\"TBEL\",\"jsScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\",\"tbelScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\"}"),
// default config for version 1 with upgrade from version 0
Arguments.of(0,
"{\"msgCount\":0,\"periodInSeconds\":1,\"originatorId\":null,\"originatorType\":null, \"scriptLang\":\"TBEL\",\"jsScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\",\"tbelScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\"}",
false,
"{\"msgCount\":0,\"periodInSeconds\":1,\"originatorId\":null,\"originatorType\":null, \"scriptLang\":\"TBEL\",\"jsScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\",\"tbelScript\":\"var msg = { temp: 42, humidity: 77 };\\nvar metadata = { data: 40 };\\nvar msgType = \\\"POST_TELEMETRY_REQUEST\\\";\\n\\nreturn { msg: msg, metadata: metadata, msgType: msgType };\"}")
);
}
@Override
protected TbNode getTestNode() {
return spy(TbMsgGeneratorNode.class);
}
}

55
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/flow/TbCheckpointNodeTest.java

@ -0,0 +1,55 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.rule.engine.flow;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.params.provider.Arguments;
import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest;
import org.thingsboard.rule.engine.api.TbNode;
import java.util.stream.Stream;
import static org.mockito.Mockito.spy;
@Slf4j
public class TbCheckpointNodeTest extends AbstractRuleNodeUpgradeTest {
// Rule nodes upgrade
private static Stream<Arguments> givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() {
return Stream.of(
// default config for version 0
Arguments.of(0,
"{\"queueName\":null}",
true,
"{}"),
// default config for version 0 with queueName
Arguments.of(0,
"{\"queueName\":\"Main\"}",
true,
"{}"),
// default config for version 1 with upgrade from version 0
Arguments.of(0,
"{}",
false,
"{}")
);
}
@Override
protected TbNode getTestNode() {
return spy(TbCheckpointNode.class);
}
}

25
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java

@ -15,7 +15,6 @@
*/ */
package org.thingsboard.rule.engine.telemetry; package org.thingsboard.rule.engine.telemetry;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
@ -25,8 +24,10 @@ import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest;
import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService;
import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNode;
import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
@ -40,7 +41,6 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.util.TbPair;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
@ -53,7 +53,6 @@ import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.BDDMockito.willCallRealMethod; import static org.mockito.BDDMockito.willCallRealMethod;
@ -65,7 +64,7 @@ import static org.mockito.Mockito.when;
import static org.thingsboard.server.common.data.DataConstants.NOTIFY_DEVICE_METADATA_KEY; import static org.thingsboard.server.common.data.DataConstants.NOTIFY_DEVICE_METADATA_KEY;
@Slf4j @Slf4j
class TbMsgAttributesNodeTest { class TbMsgAttributesNodeTest extends AbstractRuleNodeUpgradeTest {
private TenantId tenantId; private TenantId tenantId;
private DeviceId deviceId; private DeviceId deviceId;
@ -223,21 +222,9 @@ class TbMsgAttributesNodeTest {
); );
} }
@ParameterizedTest @Override
@MethodSource protected TbNode getTestNode() {
void givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig(int givenVersion, String givenConfigStr, boolean hasChanges, String expectedConfigStr) throws TbNodeException { return node;
// GIVEN
willCallRealMethod().given(node).upgrade(anyInt(), any());
JsonNode givenConfig = JacksonUtil.toJsonNode(givenConfigStr);
JsonNode expectedConfig = JacksonUtil.toJsonNode(expectedConfigStr);
// WHEN
TbPair<Boolean, JsonNode> upgradeResult = node.upgrade(givenVersion, givenConfig);
// THEN
assertThat(upgradeResult.getFirst()).isEqualTo(hasChanges);
ObjectNode upgradedConfig = (ObjectNode) upgradeResult.getSecond();
assertThat(upgradedConfig).isEqualTo(expectedConfig);
} }
} }

67
rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbMsgDeduplicationNodeTest.java

@ -22,12 +22,17 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatchers; import org.mockito.ArgumentMatchers;
import org.mockito.stubbing.Answer; import org.mockito.stubbing.Answer;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest;
import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbContext;
import org.thingsboard.rule.engine.api.TbNode;
import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeConfiguration;
import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TbNodeException;
import org.thingsboard.rule.engine.deduplication.DeduplicationStrategy; import org.thingsboard.rule.engine.deduplication.DeduplicationStrategy;
@ -55,6 +60,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Stream;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
@ -68,7 +74,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@Slf4j @Slf4j
public class TbMsgDeduplicationNodeTest { public class TbMsgDeduplicationNodeTest extends AbstractRuleNodeUpgradeTest {
private TbContext ctx; private TbContext ctx;
@ -139,13 +145,24 @@ public class TbMsgDeduplicationNodeTest {
node.destroy(); node.destroy();
} }
@Test private static Stream<Arguments> given_100_messages_strategy_first_then_verifyOutput() {
public void given_100_messages_strategy_first_then_verifyOutput() throws TbNodeException, ExecutionException, InterruptedException { return Stream.of(
Arguments.of((String) null),
Arguments.of(DataConstants.MAIN_QUEUE_NAME),
Arguments.of(DataConstants.HP_QUEUE_NAME)
);
}
@ParameterizedTest
@MethodSource
public void given_100_messages_strategy_first_then_verifyOutput(String queueName) throws TbNodeException, ExecutionException, InterruptedException {
int wantedNumberOfTellSelfInvocation = 2; int wantedNumberOfTellSelfInvocation = 2;
int msgCount = 100; int msgCount = 100;
awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation); awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation);
invokeTellSelf(wantedNumberOfTellSelfInvocation); invokeTellSelf(wantedNumberOfTellSelfInvocation);
when(ctx.getQueueName()).thenReturn(queueName);
config.setInterval(deduplicationInterval); config.setInterval(deduplicationInterval);
config.setMaxPendingMsgs(msgCount); config.setMaxPendingMsgs(msgCount);
nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config));
@ -182,6 +199,12 @@ public class TbMsgDeduplicationNodeTest {
Assertions.assertEquals(firstMsg.getData(), actualMsg.getData()); Assertions.assertEquals(firstMsg.getData(), actualMsg.getData());
Assertions.assertEquals(firstMsg.getMetaData(), actualMsg.getMetaData()); Assertions.assertEquals(firstMsg.getMetaData(), actualMsg.getMetaData());
Assertions.assertEquals(firstMsg.getType(), actualMsg.getType()); Assertions.assertEquals(firstMsg.getType(), actualMsg.getType());
if (queueName == null) {
Assertions.assertEquals(firstMsg.getQueueName(), actualMsg.getQueueName());
} else {
Assertions.assertEquals(ctx.getQueueName(), actualMsg.getQueueName());
}
} }
@Test @Test
@ -238,10 +261,10 @@ public class TbMsgDeduplicationNodeTest {
awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation); awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation);
invokeTellSelf(wantedNumberOfTellSelfInvocation); invokeTellSelf(wantedNumberOfTellSelfInvocation);
when(ctx.getQueueName()).thenReturn(DataConstants.HP_QUEUE_NAME);
config.setInterval(deduplicationInterval); config.setInterval(deduplicationInterval);
config.setStrategy(DeduplicationStrategy.ALL); config.setStrategy(DeduplicationStrategy.ALL);
config.setOutMsgType(TbMsgType.POST_ATTRIBUTES_REQUEST.name()); config.setOutMsgType(TbMsgType.POST_ATTRIBUTES_REQUEST.name());
config.setQueueName(DataConstants.HP_QUEUE_NAME);
nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config));
node.init(ctx, nodeConfiguration); node.init(ctx, nodeConfiguration);
@ -268,7 +291,7 @@ public class TbMsgDeduplicationNodeTest {
Assertions.assertEquals(getMergedData(inputMsgs), outMessage.getData()); Assertions.assertEquals(getMergedData(inputMsgs), outMessage.getData());
Assertions.assertEquals(deviceId, outMessage.getOriginator()); Assertions.assertEquals(deviceId, outMessage.getOriginator());
Assertions.assertEquals(config.getOutMsgType(), outMessage.getType()); Assertions.assertEquals(config.getOutMsgType(), outMessage.getType());
Assertions.assertEquals(config.getQueueName(), outMessage.getQueueName()); Assertions.assertEquals(DataConstants.HP_QUEUE_NAME, outMessage.getQueueName());
} }
@Test @Test
@ -278,10 +301,10 @@ public class TbMsgDeduplicationNodeTest {
awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation); awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation);
invokeTellSelf(wantedNumberOfTellSelfInvocation, true, 3); invokeTellSelf(wantedNumberOfTellSelfInvocation, true, 3);
when(ctx.getQueueName()).thenReturn(DataConstants.HP_QUEUE_NAME);
config.setInterval(deduplicationInterval); config.setInterval(deduplicationInterval);
config.setStrategy(DeduplicationStrategy.ALL); config.setStrategy(DeduplicationStrategy.ALL);
config.setOutMsgType(TbMsgType.POST_ATTRIBUTES_REQUEST.name()); config.setOutMsgType(TbMsgType.POST_ATTRIBUTES_REQUEST.name());
config.setQueueName(DataConstants.HP_QUEUE_NAME);
nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config));
node.init(ctx, nodeConfiguration); node.init(ctx, nodeConfiguration);
@ -292,7 +315,7 @@ public class TbMsgDeduplicationNodeTest {
for (TbMsg msg : firstMsgPack) { for (TbMsg msg : firstMsgPack) {
node.onMsg(ctx, msg); node.onMsg(ctx, msg);
} }
long firstPackDeduplicationPackEndTs = firstMsgPack.get(0).getMetaDataTs() + TimeUnit.SECONDS.toMillis(deduplicationInterval); long firstPackDeduplicationPackEndTs = firstMsgPack.get(0).getMetaDataTs() + TimeUnit.SECONDS.toMillis(deduplicationInterval);
List<TbMsg> secondMsgPack = getTbMsgs(deviceId, msgCount / 2, firstPackDeduplicationPackEndTs, 500); List<TbMsg> secondMsgPack = getTbMsgs(deviceId, msgCount / 2, firstPackDeduplicationPackEndTs, 500);
for (TbMsg msg : secondMsgPack) { for (TbMsg msg : secondMsgPack) {
@ -316,13 +339,13 @@ public class TbMsgDeduplicationNodeTest {
Assertions.assertEquals(getMergedData(firstMsgPack), firstMsg.getData()); Assertions.assertEquals(getMergedData(firstMsgPack), firstMsg.getData());
Assertions.assertEquals(deviceId, firstMsg.getOriginator()); Assertions.assertEquals(deviceId, firstMsg.getOriginator());
Assertions.assertEquals(config.getOutMsgType(), firstMsg.getType()); Assertions.assertEquals(config.getOutMsgType(), firstMsg.getType());
Assertions.assertEquals(config.getQueueName(), firstMsg.getQueueName()); Assertions.assertEquals(DataConstants.HP_QUEUE_NAME, firstMsg.getQueueName());
TbMsg secondMsg = resultMsgs.get(1); TbMsg secondMsg = resultMsgs.get(1);
Assertions.assertEquals(getMergedData(secondMsgPack), secondMsg.getData()); Assertions.assertEquals(getMergedData(secondMsgPack), secondMsg.getData());
Assertions.assertEquals(deviceId, secondMsg.getOriginator()); Assertions.assertEquals(deviceId, secondMsg.getOriginator());
Assertions.assertEquals(config.getOutMsgType(), secondMsg.getType()); Assertions.assertEquals(config.getOutMsgType(), secondMsg.getType());
Assertions.assertEquals(config.getQueueName(), secondMsg.getQueueName()); Assertions.assertEquals(DataConstants.HP_QUEUE_NAME, secondMsg.getQueueName());
} }
@Test @Test
@ -385,6 +408,27 @@ public class TbMsgDeduplicationNodeTest {
Assertions.assertEquals(msgWithLatestTsInSecondPack.getType(), actualMsg.getType()); Assertions.assertEquals(msgWithLatestTsInSecondPack.getType(), actualMsg.getType());
} }
// Rule nodes upgrade
private static Stream<Arguments> givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() {
return Stream.of(
// default config for version 0
Arguments.of(0,
"{\"interval\":60,\"strategy\":\"FIRST\",\"outMsgType\":null,\"maxPendingMsgs\":100,\"maxRetries\":3, \"queueName\":null}",
true,
"{\"interval\":60,\"strategy\":\"FIRST\",\"outMsgType\":null,\"maxPendingMsgs\":100,\"maxRetries\":3}"),
// default config for version 0 with queueName
Arguments.of(0,
"{\"interval\":60,\"strategy\":\"FIRST\",\"outMsgType\":null,\"maxPendingMsgs\":100,\"maxRetries\":3, \"queueName\":\"Main\"}",
true,
"{\"interval\":60,\"strategy\":\"FIRST\",\"outMsgType\":null,\"maxPendingMsgs\":100,\"maxRetries\":3}"),
// default config for version 1 with upgrade from version 0
Arguments.of(0,
"{\"interval\":60,\"strategy\":\"FIRST\",\"outMsgType\":null,\"maxPendingMsgs\":100,\"maxRetries\":3}",
false,
"{\"interval\":60,\"strategy\":\"FIRST\",\"outMsgType\":null,\"maxPendingMsgs\":100,\"maxRetries\":3}")
);
}
private TbMsg getMsgWithLatestTs(List<TbMsg> firstMsgPack) { private TbMsg getMsgWithLatestTs(List<TbMsg> firstMsgPack) {
int indexOfLastMsgInArray = firstMsgPack.size() - 1; int indexOfLastMsgInArray = firstMsgPack.size() - 1;
int indexToSetMaxTs = new Random().nextInt(indexOfLastMsgInArray) + 1; int indexToSetMaxTs = new Random().nextInt(indexOfLastMsgInArray) + 1;
@ -429,4 +473,9 @@ public class TbMsgDeduplicationNodeTest {
return JacksonUtil.toString(mergedData); return JacksonUtil.toString(mergedData);
} }
@Override
protected TbNode getTestNode() {
return node;
}
} }

1
ui-ngx/package.json

@ -97,6 +97,7 @@
"rxjs": "~7.8.0", "rxjs": "~7.8.0",
"schema-inspector": "^2.0.2", "schema-inspector": "^2.0.2",
"screenfull": "^6.0.2", "screenfull": "^6.0.2",
"sorted-btree": "^1.8.1",
"split.js": "^1.6.5", "split.js": "^1.6.5",
"systemjs": "6.14.1", "systemjs": "6.14.1",
"tinycolor2": "^1.6.0", "tinycolor2": "^1.6.0",

161
ui-ngx/src/app/core/api/data-aggregator.ts

@ -19,36 +19,40 @@ import {
IndexedSubscriptionData, IndexedSubscriptionData,
} from '@app/shared/models/telemetry/telemetry.models'; } from '@app/shared/models/telemetry/telemetry.models';
import { import {
AggregationType, AggregationType, calculateAggIntervalWithSubscriptionTimeWindow,
calculateIntervalComparisonEndTime, calculateIntervalComparisonEndTime,
calculateIntervalEndTime, calculateIntervalEndTime,
calculateIntervalStartEndTime, calculateIntervalStartEndTime,
getCurrentTime, getCurrentTime,
getTime, getTime, IntervalMath,
SubscriptionTimewindow SubscriptionTimewindow
} from '@shared/models/time/time.models'; } from '@shared/models/time/time.models';
import { UtilsService } from '@core/services/utils.service'; import { UtilsService } from '@core/services/utils.service';
import { deepClone, isDefinedAndNotNull, isNumber, isNumeric } from '@core/utils'; import { deepClone, isDefinedAndNotNull, isNumber, isNumeric } from '@core/utils';
import Timeout = NodeJS.Timeout; import Timeout = NodeJS.Timeout;
import { DataEntry, DataSet, IndexedData } from '@shared/models/widget.models';
import BTree from 'sorted-btree';
export declare type onAggregatedData = (data: IndexedSubscriptionData, detectChanges: boolean) => void; export declare type onAggregatedData = (data: IndexedData, detectChanges: boolean) => void;
interface AggData { interface AggData {
count: number; count: number;
sum: number; sum: number;
aggValue: any; aggValue: any;
ts: number;
interval: [number, number];
} }
class AggDataMap { class AggDataMap {
rangeChanged = false; private map = new BTree<number, AggData>();
private minTs = Number.MAX_SAFE_INTEGER; private reusePair: [number, AggData] = [undefined, undefined];
private map = new Map<number, AggData>();
constructor(
private subsTw: SubscriptionTimewindow,
private endTs: number
){};
set(ts: number, data: AggData) { set(ts: number, data: AggData) {
if (ts < this.minTs) {
this.rangeChanged = true;
this.minTs = ts;
}
this.map.set(ts, data); this.map.set(ts, data);
} }
@ -60,7 +64,38 @@ class AggDataMap {
this.map.delete(ts); this.map.delete(ts);
} }
forEach(callback: (value: AggData, key: number, map: Map<number, AggData>) => void, thisArg?: any) { findDataForTs(ts: number): AggData | undefined {
if (ts >= this.endTs) {
this.updateLastInterval(ts + 1);
}
const pair = this.map.getPairOrNextLower(ts, this.reusePair);
if (pair) {
const data = pair[1];
const interval = data.interval;
if (ts < interval[1]) {
return data;
}
}
}
calculateAggInterval(timestamp: number): [number, number] {
return calculateAggIntervalWithSubscriptionTimeWindow(this.subsTw, this.endTs, timestamp);
}
updateLastInterval(endTs: number) {
if (endTs > this.endTs) {
this.endTs = endTs;
const lastTs = this.map.maxKey();
if (lastTs) {
const data = this.map.get(lastTs);
const interval = calculateAggIntervalWithSubscriptionTimeWindow(this.subsTw, endTs, data.ts);
data.interval = interval;
data.ts = interval[0] + Math.floor((interval[1] - interval[0]) / 2);
}
}
}
forEach(callback: (value: AggData, key: number, map: BTree<number, AggData>) => void, thisArg?: any) {
this.map.forEach(callback, thisArg); this.map.forEach(callback, thisArg);
} }
@ -71,24 +106,6 @@ class AggDataMap {
class AggregationMap { class AggregationMap {
aggMap: {[id: number]: AggDataMap} = {}; aggMap: {[id: number]: AggDataMap} = {};
detectRangeChanged(): boolean {
let changed = false;
for (const id of Object.keys(this.aggMap)) {
const aggDataMap = this.aggMap[id];
if (aggDataMap.rangeChanged) {
changed = true;
aggDataMap.rangeChanged = false;
}
}
return changed;
}
clearRangeChangedFlags() {
for (const id of Object.keys(this.aggMap)) {
this.aggMap[id].rangeChanged = false;
}
}
} }
declare type AggFunction = (aggData: AggData, value?: any) => void; declare type AggFunction = (aggData: AggData, value?: any) => void;
@ -156,9 +173,9 @@ export class DataAggregator {
} }
} }
private dataBuffer: IndexedSubscriptionData = []; private dataBuffer: IndexedData = [];
private data: IndexedSubscriptionData; private data: IndexedData;
private readonly lastPrevKvPairData: {[id: number]: [number, any]}; private readonly lastPrevKvPairData: {[id: number]: DataEntry};
private aggregationMap: AggregationMap; private aggregationMap: AggregationMap;
@ -166,7 +183,7 @@ export class DataAggregator {
private resetPending = false; private resetPending = false;
private updatedData = false; private updatedData = false;
private aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(this.subsTw.aggregation.interval, 1000); private aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(IntervalMath.numberValue(this.subsTw.aggregation.interval), 1000);
private intervalTimeoutHandle: Timeout; private intervalTimeoutHandle: Timeout;
private intervalScheduledTime: number; private intervalScheduledTime: number;
@ -216,7 +233,7 @@ export class DataAggregator {
this.intervalScheduledTime = this.utils.currentPerfTime(); this.intervalScheduledTime = this.utils.currentPerfTime();
this.calculateStartEndTs(); this.calculateStartEndTs();
this.elapsed = 0; this.elapsed = 0;
this.aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(this.subsTw.aggregation.interval, 1000); this.aggregationTimeout = this.isLatestDataAgg ? 1000 : Math.max(IntervalMath.numberValue(this.subsTw.aggregation.interval), 1000);
this.resetPending = true; this.resetPending = true;
this.updatedData = false; this.updatedData = false;
this.intervalTimeoutHandle = setTimeout(this.onInterval.bind(this), Math.min(this.aggregationTimeout, MAX_INTERVAL_TIMEOUT)); this.intervalTimeoutHandle = setTimeout(this.onInterval.bind(this), Math.min(this.aggregationTimeout, MAX_INTERVAL_TIMEOUT));
@ -252,7 +269,6 @@ export class DataAggregator {
if (updateIntervalScheduledTime) { if (updateIntervalScheduledTime) {
this.intervalScheduledTime = this.utils.currentPerfTime(); this.intervalScheduledTime = this.utils.currentPerfTime();
} }
this.aggregationMap.clearRangeChangedFlags();
this.onInterval(history, detectChanges); this.onInterval(history, detectChanges);
} else { } else {
this.updateAggregatedData(data); this.updateAggregatedData(data);
@ -260,9 +276,7 @@ export class DataAggregator {
this.intervalScheduledTime = this.utils.currentPerfTime(); this.intervalScheduledTime = this.utils.currentPerfTime();
this.onInterval(history, detectChanges); this.onInterval(history, detectChanges);
} else { } else {
if (this.aggregationMap.detectRangeChanged()) { this.onInterval(false, detectChanges, true);
this.onInterval(false, detectChanges, true);
}
} }
} }
} }
@ -283,7 +297,7 @@ export class DataAggregator {
} }
} }
private onInterval(history?: boolean, detectChanges?: boolean, rangeChanged?: boolean) { private onInterval(history?: boolean, detectChanges?: boolean, dataChanged?: boolean) {
const now = this.utils.currentPerfTime(); const now = this.utils.currentPerfTime();
this.elapsed += now - this.intervalScheduledTime; this.elapsed += now - this.intervalScheduledTime;
this.intervalScheduledTime = now; this.intervalScheduledTime = now;
@ -291,10 +305,10 @@ export class DataAggregator {
clearTimeout(this.intervalTimeoutHandle); clearTimeout(this.intervalTimeoutHandle);
this.intervalTimeoutHandle = null; this.intervalTimeoutHandle = null;
} }
const intervalTimeout = rangeChanged ? this.aggregationTimeout - this.elapsed : this.aggregationTimeout; const intervalTimeout = dataChanged ? this.aggregationTimeout - this.elapsed : this.aggregationTimeout;
if (!history) { if (!history) {
const delta = Math.floor(this.elapsed / this.aggregationTimeout); const delta = Math.floor(this.elapsed / this.aggregationTimeout);
if (delta || !this.data || rangeChanged) { if (delta || !this.data || dataChanged) {
const tickTs = delta * this.aggregationTimeout; const tickTs = delta * this.aggregationTimeout;
if (this.subsTw.quickInterval) { if (this.subsTw.quickInterval) {
const startEndTime = calculateIntervalStartEndTime(this.subsTw.quickInterval, this.subsTw.timezone); const startEndTime = calculateIntervalStartEndTime(this.subsTw.quickInterval, this.subsTw.timezone);
@ -304,6 +318,7 @@ export class DataAggregator {
this.startTs += tickTs; this.startTs += tickTs;
this.endTs += tickTs; this.endTs += tickTs;
} }
this.updateLastInterval();
this.data = this.updateData(); this.data = this.updateData();
this.elapsed = this.elapsed - delta * this.aggregationTimeout; this.elapsed = this.elapsed - delta * this.aggregationTimeout;
} }
@ -319,7 +334,7 @@ export class DataAggregator {
} }
} }
private updateData(): IndexedSubscriptionData { private updateData(): IndexedData {
this.dataBuffer = []; this.dataBuffer = [];
this.tsKeys.forEach((key) => { this.tsKeys.forEach((key) => {
if (!this.dataBuffer[key.id]) { if (!this.dataBuffer[key.id]) {
@ -332,19 +347,21 @@ export class DataAggregator {
const aggKey = this.aggKeyById(id); const aggKey = this.aggKeyById(id);
const noAggregation = aggKey.agg === AggregationType.NONE; const noAggregation = aggKey.agg === AggregationType.NONE;
let keyData = this.dataBuffer[id]; let keyData = this.dataBuffer[id];
aggKeyData.forEach((aggData, aggTimestamp) => { const deletedKeys: number[] = [];
if (aggTimestamp < this.startTs) { aggKeyData.forEach((aggData, aggStartTs) => {
if (aggStartTs < this.startTs) {
if (this.subsTw.aggregation.stateData && if (this.subsTw.aggregation.stateData &&
(!this.lastPrevKvPairData[id] || this.lastPrevKvPairData[id][0] < aggTimestamp)) { (!this.lastPrevKvPairData[id] || this.lastPrevKvPairData[id][0] < aggData.ts)) {
this.lastPrevKvPairData[id] = [aggTimestamp, aggData.aggValue]; this.lastPrevKvPairData[id] = [aggData.ts, aggData.aggValue, aggData.interval];
} }
aggKeyData.delete(aggTimestamp); deletedKeys.push(aggStartTs);
this.updatedData = true; this.updatedData = true;
} else if (aggTimestamp < this.endTs || noAggregation) { } else if (aggData.ts < this.endTs || noAggregation) {
const kvPair: [number, any] = [aggTimestamp, aggData.aggValue]; const kvPair: DataEntry = [aggData.ts, aggData.aggValue, aggData.interval];
keyData.push(kvPair); keyData.push(kvPair);
} }
}); });
deletedKeys.forEach(ts => aggKeyData.delete(ts));
keyData.sort((set1, set2) => set1[0] - set2[0]); keyData.sort((set1, set2) => set1[0] - set2[0]);
if (this.subsTw.aggregation.stateData) { if (this.subsTw.aggregation.stateData) {
this.updateStateBounds(keyData, deepClone(this.lastPrevKvPairData[id])); this.updateStateBounds(keyData, deepClone(this.lastPrevKvPairData[id]));
@ -357,11 +374,11 @@ export class DataAggregator {
return this.dataBuffer; return this.dataBuffer;
} }
private updateStateBounds(keyData: [number, any, number?][], lastPrevKvPair: [number, any]) { private updateStateBounds(keyData: DataSet, lastPrevKvPair: DataEntry) {
if (lastPrevKvPair) { if (lastPrevKvPair) {
lastPrevKvPair[0] = this.startTs; lastPrevKvPair[0] = this.startTs;
} }
let firstKvPair; let firstKvPair: DataEntry;
if (!keyData.length) { if (!keyData.length) {
if (lastPrevKvPair) { if (lastPrevKvPair) {
firstKvPair = lastPrevKvPair; firstKvPair = lastPrevKvPair;
@ -395,20 +412,26 @@ export class DataAggregator {
const noAggregation = aggType === AggregationType.NONE; const noAggregation = aggType === AggregationType.NONE;
let aggKeyData = aggregationMap.aggMap[id]; let aggKeyData = aggregationMap.aggMap[id];
if (!aggKeyData) { if (!aggKeyData) {
aggKeyData = new AggDataMap(); aggKeyData = new AggDataMap(this.subsTw, this.endTs);
aggregationMap.aggMap[id] = aggKeyData; aggregationMap.aggMap[id] = aggKeyData;
} }
const keyData = data[id]; const keyData = data[id];
keyData.forEach((kvPair) => { keyData.forEach((kvPair) => {
const timestamp = kvPair[0]; const timestamp = kvPair[0];
const value = DataAggregator.convertValue(kvPair[1], noAggregation); const value = DataAggregator.convertValue(kvPair[1], noAggregation);
const tsKey = timestamp; let interval: [number, number] = [timestamp, timestamp];
const aggData = { if (!noAggregation) {
interval = aggKeyData.calculateAggInterval(timestamp);
}
const ts = interval[0] + Math.floor((interval[1] - interval[0]) / 2);
const aggData: AggData = {
count: isCount ? value : isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, count: isCount ? value : isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1,
sum: value, sum: value,
aggValue: value aggValue: value,
ts,
interval
}; };
aggKeyData.set(tsKey, aggData); aggKeyData.set(interval[0], aggData);
}); });
} }
return aggregationMap; return aggregationMap;
@ -423,24 +446,28 @@ export class DataAggregator {
const noAggregation = aggType === AggregationType.NONE; const noAggregation = aggType === AggregationType.NONE;
let aggKeyData = this.aggregationMap.aggMap[id]; let aggKeyData = this.aggregationMap.aggMap[id];
if (!aggKeyData) { if (!aggKeyData) {
aggKeyData = new AggDataMap(); aggKeyData = new AggDataMap(this.subsTw, this.endTs);
this.aggregationMap.aggMap[id] = aggKeyData; this.aggregationMap.aggMap[id] = aggKeyData;
} }
const keyData = data[id]; const keyData = data[id];
keyData.forEach((kvPair) => { keyData.forEach((kvPair) => {
const timestamp = kvPair[0]; const timestamp = kvPair[0];
const value = DataAggregator.convertValue(kvPair[1], noAggregation); const value = DataAggregator.convertValue(kvPair[1], noAggregation);
const aggTimestamp = noAggregation ? timestamp : (this.startTs + let aggData = aggKeyData.findDataForTs(timestamp);
Math.floor((timestamp - this.startTs) / this.subsTw.aggregation.interval) *
this.subsTw.aggregation.interval + this.subsTw.aggregation.interval / 2);
let aggData = aggKeyData.get(aggTimestamp);
if (!aggData) { if (!aggData) {
let interval: [number, number] = [timestamp, timestamp];
if (!noAggregation) {
interval = aggKeyData.calculateAggInterval(timestamp);
}
const ts = interval[0] + Math.floor((interval[1] - interval[0]) / 2);
aggData = { aggData = {
count: isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1, count: isDefinedAndNotNull(kvPair[2]) ? kvPair[2] : 1,
sum: value, sum: value,
aggValue: isCount ? 1 : value aggValue: isCount ? 1 : value,
ts,
interval
}; };
aggKeyData.set(aggTimestamp, aggData); aggKeyData.set(interval[0], aggData);
} else { } else {
DataAggregator.getAggFunction(aggType)(aggData, value); DataAggregator.getAggFunction(aggType)(aggData, value);
} }
@ -448,6 +475,14 @@ export class DataAggregator {
} }
} }
private updateLastInterval() {
for (const idStr of Object.keys(this.aggregationMap.aggMap)) {
const id = Number(idStr);
const aggKeyData = this.aggregationMap.aggMap[id];
aggKeyData.updateLastInterval(this.endTs);
}
}
private aggKeyById(id: number): AggKey { private aggKeyById(id: number): AggKey {
return this.tsKeys.find(key => key.id === id); return this.tsKeys.find(key => key.id === id);
} }

96
ui-ngx/src/app/core/api/entity-data-subscription.ts

@ -14,12 +14,19 @@
/// limitations under the License. /// limitations under the License.
/// ///
import { ComparisonResultType, DataSet, DataSetHolder, DatasourceType, widgetType } from '@shared/models/widget.models'; import {
ComparisonResultType,
DataEntry,
DataSet,
DataSetHolder,
DatasourceType, IndexedData,
widgetType
} from '@shared/models/widget.models';
import { import {
AggregationType, AggregationType,
ComparisonDuration, ComparisonDuration,
createTimewindowForComparison, createTimewindowForComparison,
getCurrentTime, getCurrentTime, IntervalMath,
SubscriptionTimewindow SubscriptionTimewindow
} from '@shared/models/time/time.models'; } from '@shared/models/time/time.models';
import { import {
@ -42,15 +49,16 @@ import {
EntityCountCmd, EntityCountCmd,
EntityDataCmd, EntityDataCmd,
IndexedSubscriptionData, IndexedSubscriptionData,
IntervalType,
NOT_SUPPORTED, NOT_SUPPORTED,
SubscriptionData, SubscriptionData, SubscriptionDataEntry,
TelemetrySubscriber TelemetrySubscriber
} from '@shared/models/telemetry/telemetry.models'; } from '@shared/models/telemetry/telemetry.models';
import { UtilsService } from '@core/services/utils.service'; import { UtilsService } from '@core/services/utils.service';
import { EntityDataListener, EntityDataLoadResult } from '@core/api/entity-data.service'; import { EntityDataListener, EntityDataLoadResult } from '@core/api/entity-data.service';
import { deepClone, isDefined, isDefinedAndNotNull, isNumeric, isObject, objectHashCode } from '@core/utils'; import { deepClone, isDefined, isDefinedAndNotNull, isNumeric, isObject, objectHashCode } from '@core/utils';
import { PageData } from '@shared/models/page/page-data'; import { PageData } from '@shared/models/page/page-data';
import { DataAggregator } from '@core/api/data-aggregator'; import { DataAggregator, onAggregatedData } from '@core/api/data-aggregator';
import { NULL_UUID } from '@shared/models/id/has-uuid'; import { NULL_UUID } from '@shared/models/id/has-uuid';
import { EntityType } from '@shared/models/entity-type.models'; import { EntityType } from '@shared/models/entity-type.models';
import { Observable, of, ReplaySubject, Subject } from 'rxjs'; import { Observable, of, ReplaySubject, Subject } from 'rxjs';
@ -154,7 +162,7 @@ export class EntityDataSubscription {
return val; return val;
} }
private static calculateComparisonValue(key: SubscriptionDataKey, comparisonTsValue: ComparisonTsValue): [number, any, number?][] { private static calculateComparisonValue(key: SubscriptionDataKey, comparisonTsValue: ComparisonTsValue): DataSet {
let timestamp: number; let timestamp: number;
let value: any; let value: any;
switch (key.comparisonResultType) { switch (key.comparisonResultType) {
@ -385,7 +393,13 @@ export class EntityDataSubscription {
if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && this.tsFields.length) { if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && this.tsFields.length) {
targetCommand.tsCmd.startTs = this.subsTw.startTs; targetCommand.tsCmd.startTs = this.subsTw.startTs;
targetCommand.tsCmd.timeWindow = this.subsTw.aggregation.timeWindow; targetCommand.tsCmd.timeWindow = this.subsTw.aggregation.timeWindow;
targetCommand.tsCmd.interval = this.subsTw.aggregation.interval; if (typeof this.subsTw.aggregation.interval === 'number') {
targetCommand.tsCmd.interval = this.subsTw.aggregation.interval;
targetCommand.tsCmd.intervalType = IntervalType.MILLISECONDS;
} else {
targetCommand.tsCmd.intervalType = this.subsTw.aggregation.interval;
}
targetCommand.tsCmd.timeZoneId = this.subsTw.timezone;
targetCommand.tsCmd.limit = this.subsTw.aggregation.limit; targetCommand.tsCmd.limit = this.subsTw.aggregation.limit;
targetCommand.tsCmd.agg = this.subsTw.aggregation.type; targetCommand.tsCmd.agg = this.subsTw.aggregation.type;
targetCommand.tsCmd.fetchLatestPreviousPoint = this.subsTw.aggregation.stateData; targetCommand.tsCmd.fetchLatestPreviousPoint = this.subsTw.aggregation.stateData;
@ -659,21 +673,35 @@ export class EntityDataSubscription {
keys: this.tsFields.map(key => key.key), keys: this.tsFields.map(key => key.key),
startTs: this.subsTw.fixedWindow.startTimeMs, startTs: this.subsTw.fixedWindow.startTimeMs,
endTs: this.subsTw.fixedWindow.endTimeMs, endTs: this.subsTw.fixedWindow.endTimeMs,
interval: this.subsTw.aggregation.interval, interval: 0,
intervalType: IntervalType.MILLISECONDS,
limit: this.subsTw.aggregation.limit, limit: this.subsTw.aggregation.limit,
timeZoneId: this.subsTw.timezone,
agg: this.subsTw.aggregation.type, agg: this.subsTw.aggregation.type,
fetchLatestPreviousPoint: this.subsTw.aggregation.stateData fetchLatestPreviousPoint: this.subsTw.aggregation.stateData
}; };
if (typeof this.subsTw.aggregation.interval === 'number') {
cmd.historyCmd.interval = this.subsTw.aggregation.interval;
} else {
cmd.historyCmd.intervalType = this.subsTw.aggregation.interval;
}
} else { } else {
cmd.tsCmd = { cmd.tsCmd = {
keys: this.tsFields.map(key => key.key), keys: this.tsFields.map(key => key.key),
startTs: this.subsTw.startTs, startTs: this.subsTw.startTs,
timeWindow: this.subsTw.aggregation.timeWindow, timeWindow: this.subsTw.aggregation.timeWindow,
interval: this.subsTw.aggregation.interval, interval: 0,
intervalType: IntervalType.MILLISECONDS,
limit: this.subsTw.aggregation.limit, limit: this.subsTw.aggregation.limit,
timeZoneId: this.subsTw.timezone,
agg: this.subsTw.aggregation.type, agg: this.subsTw.aggregation.type,
fetchLatestPreviousPoint: this.subsTw.aggregation.stateData fetchLatestPreviousPoint: this.subsTw.aggregation.stateData
}; };
if (typeof this.subsTw.aggregation.interval === 'number') {
cmd.tsCmd.interval = this.subsTw.aggregation.interval;
} else {
cmd.tsCmd.intervalType = this.subsTw.aggregation.interval;
}
} }
} }
latestValuesKeys = this.latestValues; latestValuesKeys = this.latestValues;
@ -717,7 +745,8 @@ export class EntityDataSubscription {
this.frequency = 1000; this.frequency = 1000;
this.latestFrequency = 1000; this.latestFrequency = 1000;
if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) {
this.frequency = Math.min(this.entityDataSubscriptionOptions.subscriptionTimewindow.aggregation.interval, 5000); this.frequency =
Math.min(IntervalMath.numberValue(this.entityDataSubscriptionOptions.subscriptionTimewindow.aggregation.interval), 5000);
} }
this.tickScheduledTime = this.utils.currentPerfTime(); this.tickScheduledTime = this.utils.currentPerfTime();
this.generateData(true); this.generateData(true);
@ -785,9 +814,9 @@ export class EntityDataSubscription {
} }
private reportNotSupported(keys: AggKey[], isUpdate: boolean) { private reportNotSupported(keys: AggKey[], isUpdate: boolean) {
const indexedData: IndexedSubscriptionData = []; const indexedData: IndexedData = [];
for (const key of keys) { for (const key of keys) {
indexedData[key.id] = [[0, NOT_SUPPORTED]]; indexedData[key.id] = [[0, NOT_SUPPORTED, [0,0]]];
} }
for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) { for (let dataIndex = 0; dataIndex < this.pageData.data.length; dataIndex++) {
this.onIndexedData(indexedData, dataIndex, true, this.onIndexedData(indexedData, dataIndex, true,
@ -942,7 +971,7 @@ export class EntityDataSubscription {
} }
if (Object.keys(aggData).length > 0 && this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) { if (Object.keys(aggData).length > 0 && this.tsLatestDataAggregators && this.tsLatestDataAggregators[dataIndex]) {
const dataAggregator = this.tsLatestDataAggregators[dataIndex]; const dataAggregator = this.tsLatestDataAggregators[dataIndex];
let prevDataCb; let prevDataCb: onAggregatedData;
if (!isUpdate) { if (!isUpdate) {
prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => {
this.onIndexedData(data, dataIndex, detectChanges, this.onIndexedData(data, dataIndex, detectChanges,
@ -996,7 +1025,7 @@ export class EntityDataSubscription {
for (const dataKey of dataKeys) { for (const dataKey of dataKeys) {
indexedData[dataKey.index] = subscriptionData[dataKey.name]; indexedData[dataKey.index] = subscriptionData[dataKey.name];
} }
let prevDataCb; let prevDataCb: onAggregatedData;
if (!isUpdate) { if (!isUpdate) {
prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => { prevDataCb = dataAggregator.updateOnDataCb((data, detectChanges) => {
this.onIndexedData(data, dataIndex, detectChanges, false, dataUpdatedCb); this.onIndexedData(data, dataIndex, detectChanges, false, dataUpdatedCb);
@ -1016,12 +1045,12 @@ export class EntityDataSubscription {
isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) {
for (const key of Object.keys(sourceData)) { for (const key of Object.keys(sourceData)) {
const keyData = sourceData[key]; const keyData = sourceData[key];
this.onKeyData(keyData, key, 0, type, this.onKeyData(keyData.map(entry => [entry[0], entry[1], [entry[0], entry[0]]]), key, 0, type,
dataIndex, detectChanges, isTsLatest, false, dataUpdatedCb); dataIndex, detectChanges, isTsLatest, false, dataUpdatedCb);
} }
} }
private onIndexedData(sourceData: IndexedSubscriptionData, dataIndex: number, detectChanges: boolean, private onIndexedData(sourceData: IndexedData, dataIndex: number, detectChanges: boolean,
isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) { isTsLatest: boolean, dataUpdatedCb: DataUpdatedCb) {
for (const indexStr of Object.keys(sourceData)) { for (const indexStr of Object.keys(sourceData)) {
const id = Number(indexStr); const id = Number(indexStr);
@ -1037,7 +1066,7 @@ export class EntityDataSubscription {
} }
} }
private onKeyData(keyData: [number, any, number?][], keyName: string, id: number, type: DataKeyType, private onKeyData(keyData: DataSet, keyName: string, id: number, type: DataKeyType,
dataIndex: number, detectChanges: boolean, dataIndex: number, detectChanges: boolean,
isTsLatest: boolean, isAggLatest: boolean, dataUpdatedCb: DataUpdatedCb) { isTsLatest: boolean, isAggLatest: boolean, dataUpdatedCb: DataUpdatedCb) {
const keyIdSuffix = isAggLatest ? `_${id}` : ''; const keyIdSuffix = isAggLatest ? `_${id}` : '';
@ -1048,8 +1077,8 @@ export class EntityDataSubscription {
if (this.datasourceData[dataIndex][datasourceKey].data) { if (this.datasourceData[dataIndex][datasourceKey].data) {
const dataKey = dataKeyList[keyIndex]; const dataKey = dataKeyList[keyIndex];
const data: DataSet = []; const data: DataSet = [];
let prevSeries: [number, any]; let prevSeries: DataEntry;
let prevOrigSeries: [number, any]; let prevOrigSeries: DataEntry;
let datasourceKeyData: DataSet; let datasourceKeyData: DataSet;
let datasourceOrigKeyData: DataSet; let datasourceOrigKeyData: DataSet;
let update = false; let update = false;
@ -1064,36 +1093,36 @@ export class EntityDataSubscription {
prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; prevSeries = datasourceKeyData[datasourceKeyData.length - 1];
prevOrigSeries = datasourceOrigKeyData[datasourceOrigKeyData.length - 1]; prevOrigSeries = datasourceOrigKeyData[datasourceOrigKeyData.length - 1];
} else { } else {
prevSeries = [0, 0]; prevSeries = [0, 0, [0, 0]];
prevOrigSeries = [0, 0]; prevOrigSeries = [0, 0, [0, 0]];
} }
this.datasourceOrigData[dataIndex][datasourceKey].data = []; this.datasourceOrigData[dataIndex][datasourceKey].data = [];
if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && !isTsLatest) { if (this.entityDataSubscriptionOptions.type === widgetType.timeseries && !isTsLatest) {
keyData.forEach((keySeries) => { keyData.forEach((keySeries) => {
let series = keySeries; let series = keySeries;
const time = series[0]; const time = series[0];
this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1], series[2]]);
let value = EntityDataSubscription.convertValue(series[1]); let value = EntityDataSubscription.convertValue(series[1]);
if (dataKey.postFunc) { if (dataKey.postFunc) {
value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]);
} }
prevOrigSeries = [series[0], series[1]]; prevOrigSeries = [series[0], series[1], series[2]];
series = [series[0], value]; series = [series[0], value, series[2]];
data.push([series[0], series[1]]); data.push([series[0], series[1], series[2]]);
prevSeries = [series[0], series[1]]; prevSeries = [series[0], series[1], series[2]];
}); });
update = true; update = true;
} else if (this.entityDataSubscriptionOptions.type === widgetType.latest || isTsLatest) { } else if (this.entityDataSubscriptionOptions.type === widgetType.latest || isTsLatest) {
if (keyData.length > 0) { if (keyData.length > 0) {
let series = keyData[0]; let series = keyData[0];
const time = series[0]; const time = series[0];
this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1]]); this.datasourceOrigData[dataIndex][datasourceKey].data.push([series[0], series[1], series[2]]);
let value = EntityDataSubscription.convertValue(series[1]); let value = EntityDataSubscription.convertValue(series[1]);
if (dataKey.postFunc) { if (dataKey.postFunc) {
value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]); value = dataKey.postFunc(time, value, prevSeries[1], prevOrigSeries[0], prevOrigSeries[1]);
} }
series = [time, value]; series = [time, value, series[2]];
data.push([series[0], series[1]]); data.push([series[0], series[1], series[2]]);
} }
update = true; update = true;
} }
@ -1155,13 +1184,14 @@ export class EntityDataSubscription {
return result; return result;
} }
private generateSeries(dataKey: SubscriptionDataKey, startTime: number, endTime: number): [number, any][] { private generateSeries(dataKey: SubscriptionDataKey, startTime: number, endTime: number): SubscriptionDataEntry[] {
const data: [number, any][] = []; const data: SubscriptionDataEntry[] = [];
let prevSeries: [number, any]; let prevSeries: SubscriptionDataEntry;
const datasourceDataKey = `${dataKey.key}_${dataKey.listIndex}`; const datasourceDataKey = `${dataKey.key}_${dataKey.listIndex}`;
const datasourceKeyData = this.datasourceData[0][datasourceDataKey].data; const datasourceKeyData = this.datasourceData[0][datasourceDataKey].data;
if (datasourceKeyData.length > 0) { if (datasourceKeyData.length > 0) {
prevSeries = datasourceKeyData[datasourceKeyData.length - 1]; const prevDataEntry = datasourceKeyData[datasourceKeyData.length - 1];
prevSeries = [prevDataEntry[0], prevDataEntry[1]];
} else { } else {
prevSeries = [0, 0]; prevSeries = [0, 0];
} }
@ -1178,7 +1208,7 @@ export class EntityDataSubscription {
} }
private generateLatest(dataKey: SubscriptionDataKey, detectChanges: boolean) { private generateLatest(dataKey: SubscriptionDataKey, detectChanges: boolean) {
let prevSeries: [number, any]; let prevSeries: DataEntry;
const datasourceKey = dataKey.latest ? `${dataKey.key}_${dataKey.listIndex}` : dataKey.key; const datasourceKey = dataKey.latest ? `${dataKey.key}_${dataKey.listIndex}` : dataKey.key;
const datasourceKeyData = this.datasourceData[0][datasourceKey].data; const datasourceKeyData = this.datasourceData[0][datasourceKey].data;
if (datasourceKeyData.length > 0) { if (datasourceKeyData.length > 0) {

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save