Browse Source

Merge branch 'master' into feature/java25

pull/14976/head
Viacheslav Klimov 5 months ago
committed by GitHub
parent
commit
d5f441b017
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 1
      .gitignore
  2. 2
      application/src/main/data/json/system/widget_types/bar_chart_with_labels.json
  3. 2
      application/src/main/data/json/system/widget_types/range_chart.json
  4. 58
      application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java
  5. 145
      application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java
  6. 7
      application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java
  7. 6
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java
  8. 6
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  9. 6
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java
  10. 6
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  11. 35
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextFactory.java
  12. 6
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  13. 12
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java
  14. 2
      application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java
  15. 2
      application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java
  16. 14
      application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java
  17. 10
      application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java
  18. 6
      application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java
  19. 8
      application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java
  20. 7
      application/src/main/resources/thingsboard.yml
  21. 59
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  22. 7
      application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
  23. 6
      application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java
  24. 237
      application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java
  25. 7
      application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java
  26. 17
      application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java
  27. 41
      application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java
  28. 2
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java
  29. 260
      application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java
  30. 281
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/RuleEngineConsumerLoopTest.java
  31. 2
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java
  32. 2
      application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java
  33. 1
      application/src/test/java/org/thingsboard/server/service/sms/DefaultSmsServiceTest.java
  34. 1
      application/src/test/java/org/thingsboard/server/service/stats/DevicesStatisticsTest.java
  35. 52
      application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java
  36. 60
      application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java
  37. 15
      application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java
  38. 11
      common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java
  39. 3
      common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java
  40. 7
      common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java
  41. 13
      common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java
  42. 8
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java
  43. 2
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java
  44. 5
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java
  45. 99
      common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java
  46. 2
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
  47. 4
      common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java
  48. 14
      common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java
  49. 8
      common/edge-api/src/main/proto/edge.proto
  50. 4
      common/message/src/test/java/org/thingsboard/server/common/msg/tools/TbRateLimitsTest.java
  51. 4
      common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java
  52. 20
      common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java
  53. 6
      dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java
  54. 20
      dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java
  55. 18
      dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java
  56. 14
      dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java
  57. 86
      dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java
  58. 6
      dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java
  59. 57
      dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java
  60. 24
      dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java
  61. 10
      dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java
  62. 17
      dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java
  63. 20
      dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java
  64. 13
      dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java
  65. 10
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java
  66. 9
      dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java
  67. 4
      dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java
  68. 232
      dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java
  69. 103
      dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java
  70. 56
      dao/src/test/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidatorTest.java
  71. 68
      dao/src/test/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDaoTest.java
  72. 2
      edqs/src/main/resources/edqs.yml
  73. 4
      msa/vc-executor/src/main/resources/tb-vc-executor.yml
  74. 63
      rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java
  75. 4
      transport/coap/src/main/resources/tb-coap-transport.yml
  76. 4
      transport/http/src/main/resources/tb-http-transport.yml
  77. 4
      transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml
  78. 4
      transport/mqtt/src/main/resources/tb-mqtt-transport.yml
  79. 4
      transport/snmp/src/main/resources/tb-snmp-transport.yml
  80. 51
      ui-ngx/patches/@iplab+ngx-color-picker+20.0.0.patch
  81. 31
      ui-ngx/patches/ngx-hm-carousel+19.0.0.patch
  82. 4
      ui-ngx/src/app/modules/home/components/alarm-rules/alarm-rules-table-config.ts
  83. 1
      ui-ngx/src/app/modules/home/components/api-key/api-keys-table-config.ts
  84. 2
      ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html
  85. 12
      ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts
  86. 2
      ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts
  87. 104
      ui-ngx/src/app/modules/home/components/entity/entities-table.component.html
  88. 2
      ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss
  89. 18
      ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html
  90. 12
      ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.ts
  91. 7
      ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts
  92. 6
      ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts
  93. 6
      ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts
  94. 100
      ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts
  95. 23
      ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts
  96. 11
      ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.html
  97. 45
      ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.ts
  98. 40
      ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts
  99. 6
      ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.ts
  100. 7
      ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts

1
.gitignore

@ -37,3 +37,4 @@ rebuild-docker.sh
*/.run/**
.run/**
.run
.claude/

2
application/src/main/data/json/system/widget_types/bar_chart_with_labels.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-bar-chart-with-labels-widget \n [ctx]=\"ctx\">\n</tb-bar-chart-with-labels-widget>",
"templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'humidity', label: 'Humidity', type: 'timeseries' }];\n }\n };\n}\n",
"settingsForm": [],
"dataKeySettingsForm": [],
"latestDataKeySettingsForm": [],

2
application/src/main/data/json/system/widget_types/range_chart.json

@ -11,7 +11,7 @@
"resources": [],
"templateHtml": "<tb-range-chart-widget \n [ctx]=\"ctx\">\n</tb-range-chart-widget>",
"templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n",
"controllerScript": "self.onInit = function() {\n self.ctx.$scope.rangeChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onDataUpdated();\n}\n\nself.onLatestDataUpdated = function() {\n self.ctx.$scope.rangeChartWidget.onLatestDataUpdated();\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 1,\n singleEntity: true,\n previewWidth: '80%',\n embedTitlePanel: true,\n embedActionsPanel: true,\n supportsUnitConversion: true,\n hasAdditionalLatestDataKeys: false,\n defaultDataKeysFunction: function() {\n return [{ name: 'temperature', label: 'Temperature', type: 'timeseries' }];\n }\n };\n}\n",
"settingsForm": [],
"dataKeySettingsForm": [],
"latestDataKeySettingsForm": [],

58
application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java

@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.AlarmData;
import org.thingsboard.server.common.data.query.AlarmDataQuery;
import org.thingsboard.server.common.data.query.AvailableEntityKeys;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
@ -47,6 +48,8 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.query.EntityQueryService;
import org.thingsboard.server.service.security.permission.Operation;
import java.util.Set;
import static org.thingsboard.server.controller.ControllerConstants.ALARM_DATA_QUERY_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_COUNT_QUERY_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_DATA_QUERY_DESCRIPTION;
@ -115,9 +118,11 @@ public class EntityQueryController extends BaseController {
return entityQueryService.countAlarmsByQuery(getCurrentUser(), query);
}
@Deprecated(forRemoval = true)
@ApiOperation(
value = "Find Available Entity Keys by Query",
value = "Find Available Entity Keys by Query (deprecated)",
notes = """
**Deprecated.** Use the V2 endpoint (`POST /api/v2/entitiesQuery/find/keys`) instead.\n
Returns unique time series and/or attribute key names from entities matching the query.\n
Executes the Entity Data Query to find up to 100 entities, then fetches and aggregates all distinct key names.\n
Primarily used for UI features like autocomplete suggestions.""" + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH
@ -128,9 +133,6 @@ public class EntityQueryController extends BaseController {
@Parameter(description = "Entity data query to find entities. Page size is capped at 100.")
@RequestBody EntityDataQuery query,
// fixme: combination of timeseries = false and attributes = false is allowed, but always results in empty response, therefore does not make any sense
// such combinations should NOT be allowed, but changing this will break clients
@Parameter(description = """
When true, includes unique time series key names in the response.
When false, the 'timeseries' list will be empty.""")
@ -155,6 +157,54 @@ public class EntityQueryController extends BaseController {
return wrapFuture(entityQueryService.getKeysByQuery(getCurrentUser(), getTenantId(), query, includeTimeseries, includeAttributes, scope));
}
@ApiOperation(
value = "Find Available Entity Keys By Query",
notes = """
Discovers unique time series and/or attribute key names available on entities that match the given query.
Works in two steps: first, the request body (an Entity Data Query) is executed to find matching entities
(page size is capped at 100); then, all distinct key names are collected from those entities.\n
Optionally, each key can include a sample the most recent value (by timestamp) for that key
across all matched entities."""
+ TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH
)
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@PostMapping("/v2/entitiesQuery/find/keys")
public DeferredResult<AvailableEntityKeysV2> findAvailableEntityKeysByQueryV2(
@Parameter(description = "Entity data query to find entities. Page size is capped at 100.")
@RequestBody EntityDataQuery query,
@Parameter(description = """
When true, includes unique time series keys in the response.
When false, the 'timeseries' field is omitted. At least one of 'includeTimeseries' or 'includeAttributes' must be true.""")
@RequestParam(defaultValue = "true") boolean includeTimeseries,
@Parameter(description = """
When true, includes unique attribute keys in the response.
When false, the 'attributes' field is omitted. At least one of 'includeTimeseries' or 'includeAttributes' must be true.""")
@RequestParam(defaultValue = "true") boolean includeAttributes,
@Parameter(description = """
Filters attribute keys by scope. Only applies when 'includeAttributes' is true.
When not specified, scopes are auto-determined: all three scopes (server, client, shared) for device entities,
server scope only for other entity types.""",
schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"}))
@RequestParam(required = false) Set<AttributeScope> scopes,
@Parameter(description = """
When true, each key entry includes a 'sample' object with the most recent value and timestamp.
When false, only key names are returned (sample is omitted from JSON).""")
@RequestParam(defaultValue = "false") boolean includeSamples
) throws ThingsboardException {
resolveQuery(query);
EntityDataPageLink pageLink = query.getPageLink();
if (pageLink.getPageSize() > MAX_PAGE_SIZE) {
pageLink.setPageSize(MAX_PAGE_SIZE);
}
return wrapFuture(entityQueryService.findAvailableEntityKeysByQuery(
getCurrentUser(), query,
includeTimeseries, includeAttributes, scopes, includeSamples));
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@PostMapping("/edqs/system/request")
public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) {

145
application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java

@ -15,13 +15,15 @@
*/
package org.thingsboard.server.service.query;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.KvUtil;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
@ -30,11 +32,17 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.DataType;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.AlarmData;
import org.thingsboard.server.common.data.query.AlarmDataQuery;
import org.thingsboard.server.common.data.query.AvailableEntityKeys;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2.KeyInfo;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2.KeySample;
import org.thingsboard.server.common.data.query.ComplexFilterPredicate;
import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.EntityCountQuery;
@ -59,11 +67,13 @@ import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@ -253,7 +263,7 @@ public class DefaultEntityQueryService implements EntityQueryService {
if (isAttributes) {
Map<EntityType, List<EntityId>> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType));
List<ListenableFuture<List<String>>> futures = new ArrayList<>(typesMap.size());
typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, entityIds, scope))));
typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope))));
attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> {
if (CollectionUtils.isEmpty(lists)) {
return Collections.emptyList();
@ -274,4 +284,135 @@ public class DefaultEntityQueryService implements EntityQueryService {
}, dbCallbackExecutor);
}
@Override
public ListenableFuture<AvailableEntityKeysV2> findAvailableEntityKeysByQuery(SecurityUser securityUser, EntityDataQuery query,
boolean includeTimeseries, boolean includeAttributes,
Set<AttributeScope> scopes, boolean includeSamples) {
if (!includeTimeseries && !includeAttributes) {
return Futures.immediateFailedFuture(
new IllegalArgumentException("At least one of 'includeTimeseries' or 'includeAttributes' must be true"));
}
return Futures.transformAsync(findEntityIdsByQueryAsync(securityUser, query), ids -> {
if (ids.isEmpty()) {
return immediateFuture(new AvailableEntityKeysV2(
Collections.emptySet(),
includeTimeseries ? Collections.emptyList() : null,
includeAttributes ? Collections.emptyMap() : null));
}
TenantId tenantId = securityUser.getTenantId();
Set<EntityType> entityTypes = ids.stream().map(EntityId::getEntityType).collect(Collectors.toSet());
var tsFuture = includeTimeseries ? fetchTimeseriesKeys(tenantId, ids, includeSamples) : null;
Set<AttributeScope> effectiveScopes = includeAttributes
? resolveAttributeScopes(scopes, entityTypes) : Collections.emptySet();
var attrFutures = effectiveScopes.stream()
.map(scope -> fetchAttributeKeys(tenantId, ids, scope, includeSamples))
.toList();
return assembleResult(entityTypes, tsFuture, attrFutures);
}, dbCallbackExecutor);
}
private ListenableFuture<List<EntityId>> findEntityIdsByQueryAsync(SecurityUser securityUser, EntityDataQuery query) {
return Futures.transform(entityService.findEntityDataByQueryAsync(securityUser.getTenantId(), securityUser.getCustomerId(), query),
page -> page.getData().stream()
.map(EntityData::getEntityId)
.toList(),
dbCallbackExecutor);
}
private static Set<AttributeScope> resolveAttributeScopes(Set<AttributeScope> requestedScopes, Set<EntityType> entityTypes) {
boolean hasDevices = entityTypes.contains(EntityType.DEVICE);
Set<AttributeScope> scopes;
if (CollectionUtils.isNotEmpty(requestedScopes)) {
scopes = requestedScopes;
} else { // auto-determine scopes
scopes = hasDevices
? Set.of(AttributeScope.SERVER_SCOPE, AttributeScope.CLIENT_SCOPE, AttributeScope.SHARED_SCOPE)
: Collections.singleton(AttributeScope.SERVER_SCOPE);
}
// Non-device entities only support SERVER_SCOPE
if (!hasDevices) {
return scopes.contains(AttributeScope.SERVER_SCOPE)
? Collections.singleton(AttributeScope.SERVER_SCOPE)
: Collections.emptySet();
}
return scopes;
}
private ListenableFuture<List<KeyInfo>> fetchTimeseriesKeys(TenantId tenantId, List<EntityId> entityIds, boolean includeSamples) {
if (includeSamples) {
return Futures.transform(
timeseriesService.findLatestByEntityIdsAsync(tenantId, entityIds),
entries -> toKeyInfos(entries, true),
dbCallbackExecutor);
}
return Futures.transform(
timeseriesService.findAllKeysByEntityIdsAsync(tenantId, entityIds),
keys -> keys.stream().sorted().map(k -> new KeyInfo(k, null)).toList(),
dbCallbackExecutor);
}
private ListenableFuture<Map.Entry<AttributeScope, List<KeyInfo>>> fetchAttributeKeys(
TenantId tenantId, List<EntityId> entityIds, AttributeScope scope, boolean includeSamples) {
if (includeSamples) {
return Futures.transform(
attributesService.findLatestByEntityIdsAndScopeAsync(tenantId, entityIds, scope),
entries -> Map.entry(scope, toKeyInfos(entries, true)),
dbCallbackExecutor);
}
return Futures.transform(
attributesService.findAllKeysByEntityIdsAndScopeAsync(tenantId, entityIds, scope),
keys -> Map.entry(scope, keys.stream().sorted().map(k -> new KeyInfo(k, null)).toList()),
dbCallbackExecutor);
}
private ListenableFuture<AvailableEntityKeysV2> assembleResult(
Set<EntityType> entityTypes,
ListenableFuture<List<KeyInfo>> tsFuture,
List<ListenableFuture<Map.Entry<AttributeScope, List<KeyInfo>>>> attrFutures) {
var allAttrFuture = attrFutures.isEmpty()
? immediateFuture(List.<Map.Entry<AttributeScope, List<KeyInfo>>>of())
: Futures.allAsList(attrFutures);
List<ListenableFuture<?>> allFutures = new ArrayList<>();
if (tsFuture != null) {
allFutures.add(tsFuture);
}
allFutures.add(allAttrFuture);
var finalTsFuture = tsFuture;
return Futures.whenAllComplete(allFutures)
.call(() -> {
List<KeyInfo> tsKeys = finalTsFuture != null ? Futures.getDone(finalTsFuture) : null;
Map<AttributeScope, List<KeyInfo>> attrMap = attrFutures.isEmpty() ? null : new TreeMap<>();
if (attrMap != null) {
for (var entry : Futures.getDone(allAttrFuture)) {
attrMap.put(entry.getKey(), entry.getValue());
}
}
return new AvailableEntityKeysV2(entityTypes, tsKeys, attrMap);
}, dbCallbackExecutor);
}
private static List<KeyInfo> toKeyInfos(List<? extends KvEntry> entries, boolean includeSamples) {
return entries.stream()
.map(e -> new KeyInfo(e.getKey(), includeSamples ? toKeySample(e) : null))
.sorted(Comparator.comparing(KeyInfo::key))
.toList();
}
private static KeySample toKeySample(KvEntry entry) {
long ts = entry instanceof TsKvEntry tsKv ? tsKv.getTs()
: entry instanceof AttributeKvEntry attr ? attr.getLastUpdateTs()
: 0;
JsonNode value = entry.getDataType() == DataType.JSON
? JacksonUtil.toJsonNode(entry.getJsonValue().get())
: JacksonUtil.valueToTree(entry.getValue());
return new KeySample(ts, value);
}
}

7
application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java

@ -23,11 +23,14 @@ import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.AlarmData;
import org.thingsboard.server.common.data.query.AlarmDataQuery;
import org.thingsboard.server.common.data.query.AvailableEntityKeys;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.Set;
public interface EntityQueryService {
long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query);
@ -41,4 +44,8 @@ public interface EntityQueryService {
ListenableFuture<AvailableEntityKeys> getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query,
boolean isTimeseries, boolean isAttributes, AttributeScope scope);
ListenableFuture<AvailableEntityKeysV2> findAvailableEntityKeysByQuery(SecurityUser securityUser, EntityDataQuery query,
boolean includeTimeseries, boolean includeAttributes,
Set<AttributeScope> scopes, boolean includeSamples);
}

6
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java

@ -147,15 +147,15 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa
private void processMsgs(List<TbProtoQueueMsg<ToCalculatedFieldMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldMsg>> consumer, Object consumerKey, QueueConfig config) throws Exception {
List<IdMsgPair<ToCalculatedFieldMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToCalculatedFieldMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToCalculatedFieldMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
PendingMsgHolder<ToCalculatedFieldMsg> pendingMsgHolder = new PendingMsgHolder<>();
Future<?> packSubmitFuture = consumersExecutor.submit(() -> {
orderedMsgList.forEach((element) -> {
UUID id = element.getUuid();
TbProtoQueueMsg<ToCalculatedFieldMsg> msg = element.getMsg();
UUID id = element.uuid();
TbProtoQueueMsg<ToCalculatedFieldMsg> msg = element.msg();
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {

6
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -260,15 +260,15 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private void processMsgs(List<TbProtoQueueMsg<ToCoreMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToCoreMsg>> consumer, Object consumerKey, QueueConfig config) throws Exception {
List<IdMsgPair<ToCoreMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToCoreMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToCoreMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
PendingMsgHolder<ToCoreMsg> pendingMsgHolder = new PendingMsgHolder<>();
Future<?> packSubmitFuture = consumersExecutor.submit(() -> {
orderedMsgList.forEach((element) -> {
UUID id = element.getUuid();
TbProtoQueueMsg<ToCoreMsg> msg = element.getMsg();
UUID id = element.uuid();
TbProtoQueueMsg<ToCoreMsg> msg = element.msg();
log.trace("[{}] Creating main callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {

6
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java

@ -128,15 +128,15 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService<ToEdge
private void processMsgs(List<TbProtoQueueMsg<ToEdgeMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToEdgeMsg>> consumer, Object consumerKey, QueueConfig edgeQueueConfig) throws InterruptedException {
List<IdMsgPair<ToEdgeMsg>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<ToEdgeMsg>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<ToEdgeMsg>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
PendingMsgHolder<ToEdgeMsg> pendingMsgHolder = new PendingMsgHolder<>();
Future<?> submitFuture = consumersExecutor.submit(() -> {
orderedMsgList.forEach((element) -> {
UUID id = element.getUuid();
TbProtoQueueMsg<ToEdgeMsg> msg = element.getMsg();
UUID id = element.uuid();
TbProtoQueueMsg<ToEdgeMsg> msg = element.msg();
TbCallback callback = new TbPackCallback<>(id, ctx);
try {
ToEdgeMsg toEdgeMsg = msg.getValue();

6
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -70,6 +70,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedCo
private final TbRuleEngineConsumerContext ctx;
private final QueueService queueService;
private final TbRuleEngineDeviceRpcService tbDeviceRpcService;
private final TbMsgPackProcessingContextFactory packProcessingContextFactory;
private final ConcurrentMap<QueueKey, TbRuleEngineQueueConsumerManager> consumers = new ConcurrentHashMap<>();
@ -85,11 +86,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedCo
PartitionService partitionService,
ApplicationEventPublisher eventPublisher,
JwtSettingsService jwtSettingsService,
CalculatedFieldCache calculatedFieldCache) {
CalculatedFieldCache calculatedFieldCache,
TbMsgPackProcessingContextFactory packProcessingContextFactory) {
super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, tbResourceDataCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService);
this.ctx = ctx;
this.tbDeviceRpcService = tbDeviceRpcService;
this.queueService = queueService;
this.packProcessingContextFactory = packProcessingContextFactory;
}
@Override
@ -255,6 +258,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedCo
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.packProcessingContextFactory(packProcessingContextFactory)
.build();
consumers.put(queueKey, consumer);
consumer.init(queue);

35
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextFactory.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2026 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import org.springframework.stereotype.Component;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
public interface TbMsgPackProcessingContextFactory {
TbMsgPackProcessingContext create(String queueName, TbRuleEngineSubmitStrategy submitStrategy, boolean skipTimeouts);
@Component
class DefaultTbMsgPackProcessingContextFactory implements TbMsgPackProcessingContextFactory {
@Override
public TbMsgPackProcessingContext create(String queueName, TbRuleEngineSubmitStrategy submitStrategy, boolean skipTimeouts) {
return new TbMsgPackProcessingContext(queueName, submitStrategy, skipTimeouts);
}
}
}

6
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -134,13 +134,13 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected void processNotifications(List<TbProtoQueueMsg<N>> msgs, TbQueueConsumer<TbProtoQueueMsg<N>> consumer) throws Exception {
List<IdMsgPair<N>> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList();
ConcurrentMap<UUID, TbProtoQueueMsg<N>> pendingMap = orderedMsgList.stream().collect(
Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg));
Collectors.toConcurrentMap(IdMsgPair::uuid, IdMsgPair::msg));
CountDownLatch processingTimeoutLatch = new CountDownLatch(1);
TbPackProcessingContext<TbProtoQueueMsg<N>> ctx = new TbPackProcessingContext<>(
processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>());
orderedMsgList.forEach(element -> {
UUID id = element.getUuid();
TbProtoQueueMsg<N> msg = element.getMsg();
UUID id = element.uuid();
TbProtoQueueMsg<N> msg = element.msg();
log.trace("[{}] Creating notification callback for message: {}", id, msg.getValue());
TbCallback callback = new TbPackCallback<>(id, ctx);
try {

12
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractTbRuleEngineSubmitStrategy.java

@ -44,21 +44,21 @@ public abstract class AbstractTbRuleEngineSubmitStrategy implements TbRuleEngine
@Override
public ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> getPendingMap() {
return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid, pair -> pair.msg));
return orderedMsgList.stream().collect(Collectors.toConcurrentMap(pair -> pair.uuid(), pair -> pair.msg()));
}
@Override
public void update(ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> reprocessMap) {
List<IdMsgPair<TransportProtos.ToRuleEngineMsg>> newOrderedMsgList = new ArrayList<>(reprocessMap.size());
for (IdMsgPair<TransportProtos.ToRuleEngineMsg> pair : orderedMsgList) {
if (reprocessMap.containsKey(pair.uuid)) {
if (StringUtils.isNotEmpty(pair.getMsg().getValue().getFailureMessage())) {
var toRuleEngineMsg = TransportProtos.ToRuleEngineMsg.newBuilder(pair.getMsg().getValue())
if (reprocessMap.containsKey(pair.uuid())) {
if (StringUtils.isNotEmpty(pair.msg().getValue().getFailureMessage())) {
var toRuleEngineMsg = TransportProtos.ToRuleEngineMsg.newBuilder(pair.msg().getValue())
.clearFailureMessage()
.clearRelationTypes()
.build();
var newMsg = new TbProtoQueueMsg<>(pair.getMsg().getKey(), toRuleEngineMsg, pair.getMsg().getHeaders());
newOrderedMsgList.add(new IdMsgPair<>(pair.getUuid(), newMsg));
var newMsg = new TbProtoQueueMsg<>(pair.msg().getKey(), toRuleEngineMsg, pair.msg().getHeaders());
newOrderedMsgList.add(new IdMsgPair<>(pair.uuid(), newMsg));
} else {
newOrderedMsgList.add(pair);
}

2
application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java

@ -73,7 +73,7 @@ public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS
pendingPack.clear();
for (int i = startIdx; i < endIdx; i++) {
IdMsgPair<TransportProtos.ToRuleEngineMsg> pair = orderedMsgList.get(i);
pendingPack.put(pair.uuid, pair.msg);
pendingPack.put(pair.uuid(), pair.msg());
}
tmpPack = new LinkedHashMap<>(pendingPack);
}

2
application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java

@ -34,7 +34,7 @@ public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS
if (log.isDebugEnabled()) {
log.debug("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size());
}
orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg));
orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid(), pair.msg()));
}
@Override

14
application/src/main/java/org/thingsboard/server/service/queue/processing/IdMsgPair.java

@ -15,19 +15,9 @@
*/
package org.thingsboard.server.service.queue.processing;
import lombok.Getter;
import com.google.protobuf.GeneratedMessageV3;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import java.util.UUID;
public class IdMsgPair<T extends com.google.protobuf.GeneratedMessageV3> {
@Getter
final UUID uuid;
@Getter
final TbProtoQueueMsg<T> msg;
public IdMsgPair(UUID uuid, TbProtoQueueMsg<T> msg) {
this.uuid = uuid;
this.msg = msg;
}
}
public record IdMsgPair<T extends GeneratedMessageV3>(UUID uuid, TbProtoQueueMsg<T> msg) {}

10
application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java

@ -51,7 +51,7 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs
entityIdToListMap.forEach((entityId, queue) -> {
IdMsgPair<TransportProtos.ToRuleEngineMsg> msg = queue.peek();
if (msg != null) {
msgConsumer.accept(msg.uuid, msg.msg);
msgConsumer.accept(msg.uuid(), msg.msg());
}
});
}
@ -71,13 +71,13 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs
IdMsgPair<TransportProtos.ToRuleEngineMsg> next = null;
synchronized (queue) {
IdMsgPair<TransportProtos.ToRuleEngineMsg> expected = queue.peek();
if (expected != null && expected.uuid.equals(id)) {
if (expected != null && expected.uuid().equals(id)) {
queue.poll();
next = queue.peek();
}
}
if (next != null) {
msgConsumer.accept(next.uuid, next.msg);
msgConsumer.accept(next.uuid(), next.msg());
}
}
}
@ -87,9 +87,9 @@ public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends Abs
msgToEntityIdMap.clear();
entityIdToListMap.clear();
for (IdMsgPair<TransportProtos.ToRuleEngineMsg> pair : orderedMsgList) {
EntityId entityId = getEntityId(pair.msg.getValue());
EntityId entityId = getEntityId(pair.msg().getValue());
if (entityId != null) {
msgToEntityIdMap.put(pair.uuid, entityId);
msgToEntityIdMap.put(pair.uuid(), entityId);
entityIdToListMap.computeIfAbsent(entityId, id -> new LinkedList<>()).add(pair);
}
}

6
application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java

@ -60,11 +60,11 @@ public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSu
int idx = msgIdx.get();
if (idx < listSize) {
IdMsgPair<TransportProtos.ToRuleEngineMsg> pair = orderedMsgList.get(idx);
expectedMsgId = pair.uuid;
expectedMsgId = pair.uuid();
if (log.isDebugEnabled()) {
log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg);
log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg());
}
msgConsumer.accept(pair.uuid, pair.msg);
msgConsumer.accept(pair.uuid(), pair.msg());
}
}

8
application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java

@ -42,6 +42,7 @@ import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask.Consumer
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.service.queue.TbMsgPackCallback;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContext;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory;
import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
@ -67,13 +68,15 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
private final TbRuleEngineConsumerContext ctx;
private final TbRuleEngineConsumerStats stats;
private final TbMsgPackProcessingContextFactory packProcessingContextFactory;
@Builder(builderMethodName = "create") // not to conflict with super.builder()
public TbRuleEngineQueueConsumerManager(TbRuleEngineConsumerContext ctx,
QueueKey queueKey,
ExecutorService consumerExecutor,
ScheduledExecutorService scheduler,
ExecutorService taskExecutor) {
ExecutorService taskExecutor,
TbMsgPackProcessingContextFactory packProcessingContextFactory) {
super(queueKey, null, null,
(queueConfig, tpi) -> {
Integer partitionId = tpi != null ? tpi.getPartition().orElse(-1) : null;
@ -82,6 +85,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
consumerExecutor, scheduler, taskExecutor, null);
this.ctx = ctx;
this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory());
this.packProcessingContextFactory = packProcessingContextFactory;
}
public void delete(boolean drainQueue) {
@ -134,7 +138,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager<T
TbRuleEngineProcessingStrategy ackStrategy = getProcessingStrategy(queue);
submitStrategy.init(msgs);
while (!stopped && !consumer.isStopped()) {
TbMsgPackProcessingContext packCtx = new TbMsgPackProcessingContext(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs());
TbMsgPackProcessingContext packCtx = packProcessingContextFactory.create(queue.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs());
submitStrategy.submitAttempt((id, msg) -> submitMessage(packCtx, id, msg));
final boolean timeout = !packCtx.await(queue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS);

7
application/src/main/resources/thingsboard.yml

@ -197,6 +197,8 @@ usage:
enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
# Statistics reporting interval, set to send summarized data every 10 seconds by default
interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
# Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
# Amount of statistic messages in pack
pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
check:
@ -1910,6 +1912,9 @@ queue:
print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}"
# Max length of the error message that is printed by statistics
max-error-message-length: "${TB_QUEUE_RULE_ENGINE_MAX_ERROR_MESSAGE_LENGTH:4096}"
prometheus-stats:
# Enable/disable Prometheus statistics for individual Rule Engine message processing (records time in ms for success/failure).
enabled: "${TB_QUEUE_RULE_ENGINE_PROMETHEUS_STATS_ENABLED:false}"
# After a queue is deleted (or the profile's isolation option was disabled), Rule Engine will continue reading related topics during this period before deleting the actual topics
topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}"
# Size of the thread pool that handles such operations as partition changes, config updates, queue deletion
@ -2036,7 +2041,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
health:
elasticsearch:
# Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck

59
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -123,6 +123,8 @@ import org.thingsboard.server.common.data.id.UUIDBased;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.job.Job;
import org.thingsboard.server.common.data.job.JobType;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.notification.Notification;
import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod;
import org.thingsboard.server.common.data.notification.NotificationType;
@ -177,6 +179,7 @@ import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@ -430,6 +433,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
tenantProfileService.deleteTenantProfiles(TenantId.SYS_TENANT_ID);
jdbcTemplate.execute("TRUNCATE TABLE notification");
jdbcTemplate.execute("TRUNCATE TABLE audit_log");
log.debug("Executed web test teardown");
}
@ -717,6 +721,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
return assetProfile;
}
protected Device createDevice(String name) throws Exception {
return createDevice(name, "default", null, null);
}
protected Device createDevice(String name, String accessToken) throws Exception {
return createDevice(name, "default", null, accessToken);
}
@ -730,7 +738,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration());
deviceData.setConfiguration(new DefaultDeviceConfiguration());
device.setDeviceData(deviceData);
return doPost("/api/device?accessToken=" + accessToken, device, Device.class);
if (accessToken != null) {
return doPost("/api/device?accessToken=" + accessToken, device, Device.class);
} else {
return doPost("/api/device", device, Device.class);
}
}
protected Device assignDeviceToCustomer(DeviceId deviceId, CustomerId customerId) {
@ -1218,7 +1230,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
Awaitility.await("CF state for entity actor ready to refresh dynamic arguments").atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> {
CalculatedFieldState calculatedFieldState = statesMap.get(cfId);
boolean isReady = calculatedFieldState != null && ((GeofencingCalculatedFieldState) calculatedFieldState).getLastScheduledRefreshTs() <
System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval);
System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(scheduledUpdateInterval);
log.warn("entityId {}, cfId {}, state ready to refresh == {}", entityId, cfId, isReady);
return isReady;
});
@ -1410,7 +1422,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
protected List<Job> findJobs(List<JobType> types, List<UUID> entities) throws Exception {
return doGetTypedWithPageLink("/api/jobs?types=" + types.stream().map(Enum::name).collect(Collectors.joining(",")) +
"&entities=" + entities.stream().map(UUID::toString).collect(Collectors.joining(",")) + "&",
"&entities=" + entities.stream().map(UUID::toString).collect(Collectors.joining(",")) + "&",
new TypeReference<PageData<Job>>() {}, new PageLink(100, 0, null, new SortOrder("createdTime", SortOrder.Direction.DESC))).getData();
}
@ -1424,12 +1436,37 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
protected void postTelemetry(EntityId entityId, String payload) throws Exception {
doPostAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() +
"/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk());
"/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk());
}
protected void postTelemetry(EntityId entityId, TsKvEntry entry) throws Exception {
var values = JacksonUtil.newObjectNode();
JacksonUtil.addKvEntry(values, entry);
var payload = JacksonUtil.newObjectNode()
.put("ts", entry.getTs())
.set("values", values);
var url = "/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/timeseries/any";
doPostAsync(url, payload, 30_000L).andExpect(status().isOk());
}
protected void postAttributes(EntityId entityId, AttributeScope scope, String payload) throws Exception {
doPostAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() +
"/attributes/" + scope, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk());
"/attributes/" + scope, JacksonUtil.toJsonNode(payload), 30_000L).andExpect(status().isOk());
}
protected void postAttributes(EntityId entityId, AttributeScope scope, KvEntry... attributes) throws Exception {
postAttributes(entityId, scope, Arrays.asList(attributes));
}
protected void postAttributes(EntityId entityId, AttributeScope scope, Collection<? extends KvEntry> attributes) throws Exception {
var url = "/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/attributes/" + scope;
var payload = JacksonUtil.newObjectNode();
for (KvEntry entry : attributes) {
JacksonUtil.addKvEntry(payload, entry);
}
doPostAsync(url, payload, 30_000L).andExpect(status().isOk());
}
protected CalculatedField saveCalculatedField(CalculatedField calculatedField) {
@ -1438,7 +1475,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
protected PageData<CalculatedField> getEntityCalculatedFields(EntityId entityId, CalculatedFieldType type, PageLink pageLink) throws Exception {
return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields" +
(type != null ? "?type=" + type.name() + "&" : "?"), new TypeReference<>() {}, pageLink);
(type != null ? "?type=" + type.name() + "&" : "?"), new TypeReference<>() {}, pageLink);
}
protected PageData<String> getCalculatedFieldNames(CalculatedFieldType type, PageLink pageLink) throws Exception {
@ -1451,11 +1488,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
List<UUID> entities,
List<String> names) throws Exception {
return doGetTypedWithPageLink("/api/calculatedFields?" +
(type != null ? "types=" + type + "&" : "") +
(entityType != null ? "entityType=" + entityType + "&" : "") +
(entities != null ? "entities=" + String.join(",",
entities.stream().map(UUID::toString).toList()) + "&" : "") +
(names != null ? names.stream().map(name -> "name=" + name + "&").collect(Collectors.joining("")) : ""),
(type != null ? "types=" + type + "&" : "") +
(entityType != null ? "entityType=" + entityType + "&" : "") +
(entities != null ? "entities=" + String.join(",",
entities.stream().map(UUID::toString).toList()) + "&" : "") +
(names != null ? names.stream().map(name -> "name=" + name + "&").collect(Collectors.joining("")) : ""),
new TypeReference<PageData<CalculatedFieldInfo>>() {}, new PageLink(10)).getData();
}

7
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@ -1732,11 +1732,4 @@ public class DeviceControllerTest extends AbstractControllerTest {
assertThat(fifthDevice.getName()).isEqualTo("My unique device_2");
}
private Device createDevice(String name) {
Device device = new Device();
device.setName(name);
device.setType("default");
return doPost("/api/device", device, Device.class);
}
}

6
application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java

@ -23,6 +23,7 @@ import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.edqs.EdqsState;
import org.thingsboard.server.common.data.edqs.EdqsState.EdqsApiMode;
import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest;
import org.awaitility.core.ThrowingRunnable;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.AlarmData;
@ -86,6 +87,11 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest {
result -> result == expectedResult);
}
@Override
protected void verifyAvailableKeysByQueryV2(ThrowingRunnable assertion) {
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(assertion);
}
@Test
public void testEdqsState() throws Exception {
loginSysAdmin();

237
application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@ -17,7 +17,11 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.BooleanNode;
import com.fasterxml.jackson.databind.node.DoubleNode;
import com.fasterxml.jackson.databind.node.IntNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.awaitility.core.ThrowingRunnable;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -43,12 +47,21 @@ import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.AlarmData;
import org.thingsboard.server.common.data.query.AlarmDataPageLink;
import org.thingsboard.server.common.data.query.AlarmDataQuery;
import org.thingsboard.server.common.data.query.AliasEntityId;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2.KeyInfo;
import org.thingsboard.server.common.data.query.DeviceTypeFilter;
import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.DynamicValueSourceType;
@ -84,6 +97,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
@ -1329,7 +1343,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
//assign dashboard
doPost("/api/customer/" + savedCustomer.getId().getId().toString()
+ "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
+ "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
// check entity data query by customer
User customerUser = new User();
@ -1454,6 +1468,10 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
return result;
}
protected void verifyAvailableKeysByQueryV2(ThrowingRunnable assertion) throws Throwable {
assertion.run();
}
private KeyFilter getEntityFieldEqualFilter(String keyName, String value) {
return getEntityFieldKeyFilter(keyName, value, StringFilterPredicate.StringOperation.EQUAL);
}
@ -1494,4 +1512,221 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
return nameFilter;
}
// --- findAvailableEntityKeysV2 tests ---
@Test
public void testFindAvailableKeysByQueryV2() throws Throwable {
// GIVEN — two devices matched by query; a third device should not be matched
var device1 = createDevice("Test device 1");
var device2 = createDevice("Test device 2");
var unmatchedDevice = createDevice("Unmatched device");
// unmatched device has unique keys that must NOT appear in the result
postTelemetry(unmatchedDevice.getId(), new BasicTsKvEntry(9000, new DoubleDataEntry("unmatchedTs", 999.0)));
postAttributes(unmatchedDevice.getId(), AttributeScope.SHARED_SCOPE, new StringDataEntry("unmatchedAttr", "nope"));
// device1: timeseries1 (Double) with two data points, and timeseries2 older data point
postTelemetry(device1.getId(), new BasicTsKvEntry(1000, new DoubleDataEntry("timeseries1", 10.0)));
postTelemetry(device1.getId(), new BasicTsKvEntry(2000, new DoubleDataEntry("timeseries1", 20.5)));
postTelemetry(device1.getId(), new BasicTsKvEntry(1000, new LongDataEntry("timeseries2", 100L)));
// device2: timeseries2 (Long) with a newer data point, and timeseries3 only on this device
postTelemetry(device2.getId(), new BasicTsKvEntry(3000, new LongDataEntry("timeseries2", 300L)));
postTelemetry(device2.getId(), new BasicTsKvEntry(5000, new DoubleDataEntry("timeseries3", 99.9)));
// device1: SHARED_SCOPE attributes
postAttributes(device1.getId(), AttributeScope.SHARED_SCOPE,
new BooleanDataEntry("sharedAttribute1", true), new DoubleDataEntry("sharedAttribute2", 3.14));
// device2: CLIENT_SCOPE attributes (saved via service to bypass API restriction)
attributesService.save(tenantId, device2.getId(), AttributeScope.CLIENT_SCOPE, List.of(
new BaseAttributeKvEntry(new JsonDataEntry("clientAttribute1", "{\"key\":\"val\"}"), System.currentTimeMillis()),
new BaseAttributeKvEntry(new BooleanDataEntry("clientAttribute2", false), System.currentTimeMillis())
)).get();
// device1 also has SERVER_SCOPE attributes (should be omitted by scope filter)
postAttributes(device1.getId(), AttributeScope.SERVER_SCOPE,
new StringDataEntry("serverAttribute1", "sv1"), new LongDataEntry("serverAttribute2", 42L));
// WHEN — query matches both devices; request timeseries + only SHARED and CLIENT attribute scopes
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceTypes(List.of("default"));
filter.setDeviceNameFilter("Test device");
EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, null);
EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(), null, null);
// THEN
verifyAvailableKeysByQueryV2(() -> {
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(query,
true, true, List.of(AttributeScope.SHARED_SCOPE, AttributeScope.CLIENT_SCOPE), true);
assertThat(result.entityTypes()).containsExactly(EntityType.DEVICE);
// timeseries: keys collected from both devices, samples contain the freshest data points
assertThat(result.timeseries()).extracting(KeyInfo::key)
.containsExactly("timeseries1", "timeseries2", "timeseries3");
assertThat(result.timeseries()).allSatisfy(ki -> assertThat(ki.sample()).isNotNull());
assertKeySample(result.timeseries(), "timeseries1", new DoubleNode(20.5), 2000); // from device1
assertKeySample(result.timeseries(), "timeseries2", new IntNode(300), 3000); // from device2 (newer)
assertKeySample(result.timeseries(), "timeseries3", new DoubleNode(99.9), 5000); // only on device2
// SERVER_SCOPE must be fully omitted from the response
assertThat(result.attributes()).containsOnlyKeys(AttributeScope.SHARED_SCOPE, AttributeScope.CLIENT_SCOPE);
// SHARED_SCOPE: from device1 (alphabetical order)
assertThat(result.attributes().get(AttributeScope.SHARED_SCOPE))
.extracting(KeyInfo::key).containsExactly("sharedAttribute1", "sharedAttribute2");
assertKeySample(result.attributes().get(AttributeScope.SHARED_SCOPE), "sharedAttribute1", BooleanNode.TRUE);
assertKeySample(result.attributes().get(AttributeScope.SHARED_SCOPE), "sharedAttribute2", new DoubleNode(3.14));
// CLIENT_SCOPE: from device2 (alphabetical order)
assertThat(result.attributes().get(AttributeScope.CLIENT_SCOPE))
.extracting(KeyInfo::key).containsExactly("clientAttribute1", "clientAttribute2");
assertKeySample(result.attributes().get(AttributeScope.CLIENT_SCOPE), "clientAttribute1", JacksonUtil.toJsonNode("{\"key\":\"val\"}"));
assertKeySample(result.attributes().get(AttributeScope.CLIENT_SCOPE), "clientAttribute2", BooleanNode.FALSE);
});
}
@Test
public void testFindAvailableKeysByQueryV2_withoutSamples() throws Throwable {
// GIVEN
var device = createDevice("Test device");
postTelemetry(device.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", 10.0)));
postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("firmware", "v1.0"));
// THEN
verifyAvailableKeysByQueryV2(() -> {
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(
buildDeviceQuery("Test device"), true, true, null, false);
assertThat(result.timeseries()).allSatisfy(ki -> assertThat(ki.sample()).isNull());
assertThat(result.attributes().get(AttributeScope.SERVER_SCOPE))
.allSatisfy(ki -> assertThat(ki.sample()).isNull());
});
}
@Test
public void testFindAvailableKeysByQueryV2_timeseriesOnly() throws Throwable {
// GIVEN
var device = createDevice("Test device");
postTelemetry(device.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", 10.0)));
postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("firmware", "v1.0"));
// THEN
verifyAvailableKeysByQueryV2(() -> {
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(
buildDeviceQuery("Test device"), true, false, null, false);
assertThat(result.timeseries()).extracting(KeyInfo::key).contains("temperature");
assertThat(result.attributes()).isNull();
});
}
@Test
public void testFindAvailableKeysByQueryV2_attributesOnly() throws Throwable {
// GIVEN
var device = createDevice("Test device");
postTelemetry(device.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new DoubleDataEntry("temperature", 10.0)));
postAttributes(device.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("firmware", "v1.0"));
// THEN
verifyAvailableKeysByQueryV2(() -> {
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(
buildDeviceQuery("Test device"), false, true, null, false);
assertThat(result.timeseries()).isNull();
assertThat(result.attributes().get(AttributeScope.SERVER_SCOPE))
.extracting(KeyInfo::key).contains("firmware");
});
}
@Test
public void testFindAvailableKeysByQueryV2_noMatchingEntities() throws Throwable {
// THEN
verifyAvailableKeysByQueryV2(() -> {
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(
buildDeviceQuery("NonExistentDevice_" + UUID.randomUUID()), true, true, null, true);
assertThat(result.entityTypes()).isEmpty();
assertThat(result.timeseries()).isEmpty();
assertThat(result.attributes()).isEmpty();
});
}
@Test
public void testFindAvailableKeysByQueryV2_assetUsesServerScopeOnly() throws Throwable {
// GIVEN
var asset = new Asset();
asset.setName("Test asset");
asset.setType("default");
asset = doPost("/api/asset", asset, Asset.class);
postAttributes(asset.getId(), AttributeScope.SERVER_SCOPE, new StringDataEntry("location", "warehouse"));
// WHEN
var filter = new SingleEntityFilter();
filter.setSingleEntity(AliasEntityId.fromEntityId(asset.getId()));
var query = new EntityDataQuery(filter, new EntityDataPageLink(1, 0, null, null), Collections.emptyList(), null, null);
// THEN
verifyAvailableKeysByQueryV2(() -> {
AvailableEntityKeysV2 result = findAvailableEntityKeysByQueryV2(query, false, true, null, false);
assertThat(result.entityTypes()).containsExactly(EntityType.ASSET);
assertThat(result.attributes()).containsOnlyKeys(AttributeScope.SERVER_SCOPE);
assertThat(result.attributes().get(AttributeScope.SERVER_SCOPE))
.extracting(KeyInfo::key).containsExactly("location");
});
}
@Test
public void testFindAvailableKeysByQueryV2_rejectsWhenNoKeyTypeRequested() throws Exception {
// WHEN / THEN
EntityDataQuery query = buildDeviceQuery("NonExistent");
doPostAsync("/api/v2/entitiesQuery/find/keys?includeTimeseries=false&includeAttributes=false",
query, 30_000L).andExpect(status().isBadRequest());
}
protected AvailableEntityKeysV2 findAvailableEntityKeysByQueryV2(EntityDataQuery query,
boolean includeTimeseries, boolean includeAttributes,
List<AttributeScope> scopes, boolean includeSamples) throws Exception {
StringBuilder url = new StringBuilder("/api/v2/entitiesQuery/find/keys?")
.append("includeTimeseries=").append(includeTimeseries)
.append("&includeAttributes=").append(includeAttributes)
.append("&includeSamples=").append(includeSamples);
if (scopes != null) {
for (AttributeScope scope : scopes) {
url.append("&scopes=").append(scope);
}
}
return doPostAsyncWithTypedResponse(url.toString(), query,
new TypeReference<>() {}, status().isOk());
}
private static void assertKeySample(List<KeyInfo> keys, String expectedKey, JsonNode expectedValue, long expectedTs) {
KeyInfo keyInfo = findKeyInfo(keys, expectedKey);
assertThat(keyInfo.sample()).isNotNull();
assertThat(keyInfo.sample().value()).isEqualTo(expectedValue);
assertThat(keyInfo.sample().ts()).isEqualTo(expectedTs);
}
private static void assertKeySample(List<KeyInfo> keys, String expectedKey, JsonNode expectedValue) {
KeyInfo keyInfo = findKeyInfo(keys, expectedKey);
assertThat(keyInfo.sample()).isNotNull();
assertThat(keyInfo.sample().value()).isEqualTo(expectedValue);
assertThat(keyInfo.sample().ts()).isGreaterThan(0);
}
private static KeyInfo findKeyInfo(List<KeyInfo> keys, String key) {
return keys.stream()
.filter(ki -> ki.key().equals(key)).findFirst().orElseThrow();
}
private static EntityDataQuery buildDeviceQuery(String deviceName) {
var filter = new DeviceTypeFilter();
filter.setDeviceTypes(Collections.singletonList("default"));
filter.setDeviceNameFilter(deviceName);
return new EntityDataQuery(filter, new EntityDataPageLink(1, 0, null, null), Collections.emptyList(), null, null);
}
}

7
application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java

@ -633,13 +633,6 @@ public class EntityRelationControllerTest extends AbstractControllerTest {
deleteDifferentTenant();
}
private Device createDevice(String name) {
var device = new Device();
device.setName(name);
device.setType("default");
return doPost("/api/device", device, Device.class);
}
private ResultActions getRelation(EntityRelation relation) throws Exception {
return doGet("/api/relation?" +
"fromId=" + relation.getFrom().getId() +

17
application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java

@ -407,20 +407,15 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
testBroadcastEntityStateChangeEventNeverTenantProfile();
}
private void awaitAuditLog(String awaitMessage, TenantProfileId tenantProfileId, ActionType expectedAction) throws Exception {
private void awaitAuditLog(String awaitMessage, TenantProfileId tenantProfileId, ActionType expectedAction) {
Awaitility.await(awaitMessage)
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() ->
doGetTypedWithTimePageLink(
"/api/audit/logs/entity/TENANT_PROFILE/" + tenantProfileId.getId() + "?",
new TypeReference<PageData<AuditLog>>() {
},
new TimePageLink(5))
.getData()
.stream()
.anyMatch(log -> log.getActionType() == expectedAction)
.until(() -> doGetTypedWithTimePageLink(
"/api/audit/logs?",
new TypeReference<PageData<AuditLog>>() {},
new TimePageLink(100)).getData().stream()
.anyMatch(log -> log.getEntityId().equals(tenantProfileId) && log.getActionType() == expectedAction)
);
}
private TenantProfile createTenantProfile(String name) {

41
application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java

@ -19,6 +19,8 @@ import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.ApiUsageStateValue;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest;
@ -30,6 +32,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.controller.TbUrlConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;
@ -46,6 +49,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@TestPropertySource(properties = {
"usage.stats.report.enabled=true",
"usage.stats.report.interval=2",
"usage.stats.report.urgent_interval=1",
"usage.stats.gauge_report_interval=1",
})
public class ApiUsageTest extends AbstractControllerTest {
@ -54,9 +58,12 @@ public class ApiUsageTest extends AbstractControllerTest {
private User tenantAdmin;
private static final int MAX_DP_ENABLE_VALUE = 12;
private static final int MAX_SMS_ENABLE_VALUE = 10;
private static final double WARN_THRESHOLD_VALUE = 0.5;
@Autowired
private ApiUsageStateService apiUsageStateService;
@Autowired
private TbApiUsageReportClient apiUsageReportClient;
@Before
public void beforeTest() throws Exception {
@ -82,7 +89,7 @@ public class ApiUsageTest extends AbstractControllerTest {
}
@Test
public void testTelemetryApiCall() throws Exception {
public void testDbStorageApiUsage() throws Exception {
Device device = createDevice();
assertNotNull(device);
String telemetryPayload = "{\"temperature\":25, \"humidity\":60}";
@ -94,7 +101,8 @@ public class ApiUsageTest extends AbstractControllerTest {
doPostAsync(url, telemetryPayload, String.class, status().isOk());
}
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(ApiUsageStateValue.WARNING, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState()));
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() ->
assertEquals(ApiUsageStateValue.WARNING, getUsageState().getDbStorageState()));
long VALUE_DISABLE = (long) (MAX_DP_ENABLE_VALUE - (MAX_DP_ENABLE_VALUE * WARN_THRESHOLD_VALUE)) / 2;
@ -104,10 +112,35 @@ public class ApiUsageTest extends AbstractControllerTest {
await().atMost(TIMEOUT, TimeUnit.SECONDS)
.untilAsserted(() -> {
assertEquals(ApiUsageStateValue.DISABLED, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState());
assertEquals(ApiUsageStateValue.DISABLED, getUsageState().getDbStorageState());
});
}
@Test
public void testSmsApiUsage() {
long smsWarnThreshold = (long) (MAX_SMS_ENABLE_VALUE * WARN_THRESHOLD_VALUE);
for (int i = 0; i < smsWarnThreshold; i++) {
apiUsageReportClient.report(tenantId, null, ApiUsageRecordKey.SMS_EXEC_COUNT);
}
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() ->
assertEquals(ApiUsageStateValue.WARNING, getUsageState().getSmsExecState()));
long smsDisableCount = MAX_SMS_ENABLE_VALUE - smsWarnThreshold;
for (int i = 0; i < smsDisableCount; i++) {
apiUsageReportClient.report(tenantId, null, ApiUsageRecordKey.SMS_EXEC_COUNT);
}
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() ->
assertEquals(ApiUsageStateValue.DISABLED, getUsageState().getSmsExecState()));
}
private ApiUsageState getUsageState() {
return apiUsageStateService.findTenantApiUsageState(tenantId);
}
private TenantProfile createTenantProfile() {
TenantProfile tenantProfile = new TenantProfile();
tenantProfile.setName("Tenant Profile");
@ -116,6 +149,8 @@ public class ApiUsageTest extends AbstractControllerTest {
TenantProfileData tenantProfileData = new TenantProfileData();
DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder()
.maxDPStorageDays(MAX_DP_ENABLE_VALUE)
.maxSms(MAX_SMS_ENABLE_VALUE)
.smsEnabled(true)
.warnThreshold(WARN_THRESHOLD_VALUE)
.build();

2
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/RelatedEntitiesAggregationCalculatedFieldStateTest.java

@ -234,6 +234,8 @@ public class RelatedEntitiesAggregationCalculatedFieldStateTest {
config.setUseLatestTs(true);
config.setScheduledUpdateInterval(10);
calculatedField.setConfiguration(config);
calculatedField.setVersion(1L);
return calculatedField;

260
application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java

@ -15,14 +15,17 @@
*/
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import com.google.common.util.concurrent.MoreExecutors;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
@ -35,30 +38,241 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertTrue;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.BDDMockito.then;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(MockitoJUnitRunner.class)
public class TbMsgPackProcessingContextTest {
@ExtendWith(MockitoExtension.class)
class TbMsgPackProcessingContextTest {
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
@Mock
TbRuleEngineSubmitStrategy submitStrategy;
@Mock
TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg> mockMsg;
ConcurrentMap<UUID, TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> pendingMap;
public static final int TIMEOUT = 10;
ExecutorService executorService;
@After
public void tearDown() {
@BeforeEach
void setup() {
pendingMap = new ConcurrentHashMap<>();
lenient().when(submitStrategy.getPendingMap()).thenReturn(pendingMap);
}
@AfterEach
void tearDown() {
if (executorService != null) {
executorService.shutdownNow();
MoreExecutors.shutdownAndAwaitTermination(executorService, 5, TimeUnit.SECONDS);
}
}
@Test
public void testHighConcurrencyCase() throws InterruptedException {
//log.warn("preparing the test...");
void testAwait_shouldReturnTrue_whenOnSuccessIsCalledBeforeTimeout() throws InterruptedException {
// GIVEN - a context with one pending message
executorService = Executors.newSingleThreadExecutor();
UUID msgId = UUID.randomUUID();
pendingMap.put(msgId, mockMsg);
var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false);
// WHEN - onSuccess() is called in another thread before timeout
executorService.submit(() -> {
try {
Thread.sleep(100);
context.onSuccess(msgId);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
// THEN - await() should return true (successful completion)
boolean result = context.await(5000, TimeUnit.MILLISECONDS);
assertThat(result).as("await() should return true when latch is counted down before timeout").isTrue();
// Verify the message was moved to success map
assertThat(context.getSuccessMap()).containsKey(msgId);
assertThat(context.getPendingMap()).isEmpty();
assertThat(context.getExceptionsMap()).isEmpty();
// Verify submit strategy was notified about successful message processing
then(submitStrategy).should().onSuccess(msgId);
}
@Test
void testAwait_shouldReturnTrue_whenOnFailureIsCalledBeforeTimeout() throws InterruptedException {
// GIVEN - a context with one pending message
executorService = Executors.newSingleThreadExecutor();
UUID msgId = UUID.randomUUID();
pendingMap.put(msgId, mockMsg);
var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false);
var exception = new RuleEngineException("Test exception");
// WHEN - onFailure() is called in another thread before timeout
executorService.submit(() -> {
try {
Thread.sleep(100);
context.onFailure(tenantId, msgId, exception);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
// THEN - await() should return true (successful completion, even if message processing failed)
boolean result = context.await(5000, TimeUnit.MILLISECONDS);
assertThat(result).as("await() should return true when latch is counted down before timeout").isTrue();
// Verify the exception was added to exceptions map
assertThat(context.getSuccessMap()).isEmpty();
assertThat(context.getPendingMap()).isEmpty();
assertThat(context.getExceptionsMap()).containsEntry(tenantId, exception);
}
@Test
void testAwait_shouldReturnFalse_whenTimeoutOccurs() throws InterruptedException {
// GIVEN - a context with one pending message and no processing
UUID msgId = UUID.randomUUID();
pendingMap.put(msgId, mockMsg);
var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false);
// WHEN - await() is called with short timeout and no message processing happens
long startTime = System.nanoTime();
boolean result = context.await(100, TimeUnit.MILLISECONDS);
long elapsedTime = System.nanoTime() - startTime;
// THEN - await() should return false (timeout occurred)
assertThat(result).as("await() should return false when timeout occurs").isFalse();
assertThat(elapsedTime).as("await() should wait for at least the timeout duration").isGreaterThanOrEqualTo(100L);
// Message should still be in pending map
assertThat(context.getSuccessMap()).isEmpty();
assertThat(context.getPendingMap()).containsKey(msgId);
assertThat(context.getExceptionsMap()).isEmpty();
}
@Test
void testAwait_shouldHandleMultiplePendingMessages() throws InterruptedException {
// GIVEN - a context with multiple pending messages
executorService = Executors.newSingleThreadExecutor();
UUID msgId1 = UUID.randomUUID();
UUID msgId2 = UUID.randomUUID();
UUID msgId3 = UUID.randomUUID();
pendingMap.put(msgId1, mockMsg);
pendingMap.put(msgId2, mockMsg);
pendingMap.put(msgId3, mockMsg);
var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false);
// WHEN - messages are processed one by one
executorService.submit(() -> {
try {
Thread.sleep(50);
context.onSuccess(msgId1);
Thread.sleep(50);
context.onSuccess(msgId2);
Thread.sleep(50);
context.onSuccess(msgId3);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
// THEN - await() should return true only after all messages are processed
boolean result = context.await(5000, TimeUnit.MILLISECONDS);
assertThat(result).as("await() should return true after all messages are processed").isTrue();
// All messages should be in success map
assertThat(context.getSuccessMap()).containsKeys(msgId1, msgId2, msgId3);
assertThat(context.getPendingMap()).isEmpty();
assertThat(context.getExceptionsMap()).isEmpty();
}
@Test
void testAwait_shouldNotCountDownPrematurely_withMultipleMessages() throws InterruptedException {
// GIVEN - a context with multiple pending messages
executorService = Executors.newSingleThreadExecutor();
UUID msgId1 = UUID.randomUUID();
UUID msgId2 = UUID.randomUUID();
pendingMap.put(msgId1, mockMsg);
pendingMap.put(msgId2, mockMsg);
var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false);
// WHEN - only one message is processed
executorService.submit(() -> {
try {
Thread.sleep(100);
context.onSuccess(msgId1);
// msgId2 still in processing
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
// THEN: await should timeout because not all messages were processed
boolean result = context.await(2000, TimeUnit.MILLISECONDS);
assertThat(result).as("await() should timeout when not all messages are processed").isFalse();
// One message in success, one still pending
assertThat(context.getSuccessMap()).containsOnlyKeys(msgId1);
assertThat(context.getPendingMap()).containsOnlyKeys(msgId2);
assertThat(context.getExceptionsMap()).isEmpty();
}
@Test
void testAwait_shouldHandleMixedSuccessAndFailure() throws InterruptedException {
// GIVEN - multiple messages
executorService = Executors.newSingleThreadExecutor();
UUID msgId1 = UUID.randomUUID();
UUID msgId2 = UUID.randomUUID();
pendingMap.put(msgId1, mockMsg);
pendingMap.put(msgId2, mockMsg);
var context = new TbMsgPackProcessingContext("test-queue", submitStrategy, false);
var exception = new RuleEngineException("Test exception");
// WHEN - one succeeds, one fails
executorService.submit(() -> {
try {
Thread.sleep(50);
context.onSuccess(msgId1);
Thread.sleep(50);
context.onFailure(tenantId, msgId2, exception);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
// THEN - await() should complete successfully
boolean result = context.await(5000, TimeUnit.MILLISECONDS);
assertThat(result).as("await() should return true when all messages are processed").isTrue();
assertThat(context.getSuccessMap()).containsOnlyKeys(msgId1);
assertThat(context.getPendingMap()).isEmpty();
assertThat(context.getExceptionsMap()).containsEntry(tenantId, exception);
}
@Test
void testHighConcurrencyCase() throws InterruptedException {
int msgCount = 1000;
int parallelCount = 5;
executorService = Executors.newFixedThreadPool(parallelCount, ThingsBoardThreadFactory.forName(getClass().getSimpleName() + "-test-scope"));
@ -76,28 +290,24 @@ public class TbMsgPackProcessingContextTest {
final CountDownLatch startLatch = new CountDownLatch(1);
final CountDownLatch finishLatch = new CountDownLatch(parallelCount);
for (int i = 0; i < parallelCount; i++) {
//final String taskName = "" + uuid + " " + i;
executorService.submit(() -> {
//log.warn("ready {}", taskName);
readyLatch.countDown();
try {
startLatch.await();
} catch (InterruptedException e) {
Assert.fail("failed to await");
fail("failed to await");
}
//log.warn("go {}", taskName);
context.onSuccess(uuid);
finishLatch.countDown();
});
}
assertTrue(readyLatch.await(TIMEOUT, TimeUnit.SECONDS));
assertTrue(readyLatch.await(10, TimeUnit.SECONDS));
Thread.yield();
startLatch.countDown(); //run all-at-once submitted tasks
assertTrue(finishLatch.await(TIMEOUT, TimeUnit.SECONDS));
assertTrue(finishLatch.await(10, TimeUnit.SECONDS));
}
assertTrue(context.await(TIMEOUT, TimeUnit.SECONDS));
assertTrue(context.await(10, TimeUnit.SECONDS));
verify(strategyMock, times(msgCount)).onSuccess(any(UUID.class));
}
}

281
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/RuleEngineConsumerLoopTest.java

@ -0,0 +1,281 @@
/**
* Copyright © 2016-2026 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue.ruleengine;
import com.google.common.util.concurrent.MoreExecutors;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.ProcessingStrategyType;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueMsg;
import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.memory.DefaultInMemoryStorage;
import org.thingsboard.server.queue.memory.InMemoryStorage;
import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContext;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import java.time.Duration;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class RuleEngineConsumerLoopTest {
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID());
DeviceId deviceId = new DeviceId(UUID.randomUUID());
InMemoryStorage storage;
@Mock
ActorSystemContext actorContext;
@Mock
StatsFactory statsFactory;
@Mock
TbRuleEngineQueueFactory queueFactory;
@Mock
RuleEngineStatisticsService statisticsService;
@Mock
TbServiceInfoProvider serviceInfoProvider;
@Mock
PartitionService partitionService;
@Mock
TbQueueProducerProvider producerProvider;
@Mock
TbQueueAdmin queueAdmin;
@Mock
TbMsgPackProcessingContextFactory packProcessingContextFactory;
@Mock
TbMsgPackProcessingContext packCtx;
Queue mainQueue;
TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToRuleEngineMsg>> consumer;
TbRuleEngineConsumerContext ruleEngineConsumerContext;
TbRuleEngineQueueConsumerManager consumerManager;
ExecutorService consumersExecutor;
ScheduledExecutorService scheduler;
ExecutorService mgmtExecutor;
@BeforeEach
void setup() throws InterruptedException {
consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer"));
scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("tb-rule-engine-consumer-scheduler");
mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(1, "tb-rule-engine-mgmt");
mainQueue = new Queue();
mainQueue.setTenantId(TenantId.SYS_TENANT_ID);
mainQueue.setName("Main");
mainQueue.setTopic("tb_rule_engine.main");
mainQueue.setPollInterval(25);
mainQueue.setPartitions(1);
mainQueue.setConsumerPerPartition(false);
mainQueue.setPackProcessingTimeout(2000L);
var submitStrategy = new SubmitStrategy();
submitStrategy.setType(SubmitStrategyType.BURST);
submitStrategy.setBatchSize(1000);
mainQueue.setSubmitStrategy(submitStrategy);
var processingStrategy = new ProcessingStrategy();
processingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES);
processingStrategy.setRetries(3);
processingStrategy.setFailurePercentage(0.0);
processingStrategy.setPauseBetweenRetries(3);
processingStrategy.setMaxPauseBetweenRetries(3);
mainQueue.setProcessingStrategy(processingStrategy);
storage = new DefaultInMemoryStorage();
consumer = spy(new InMemoryTbQueueConsumer<>(storage, mainQueue.getTopic()));
given(queueFactory.createToRuleEngineMsgConsumer(eq(mainQueue), isNull())).willReturn(consumer);
ruleEngineConsumerContext = new TbRuleEngineConsumerContext(
actorContext, statsFactory, new TbRuleEngineSubmitStrategyFactory(), new TbRuleEngineProcessingStrategyFactory(),
queueFactory, statisticsService, serviceInfoProvider, partitionService, producerProvider, queueAdmin
);
ruleEngineConsumerContext.setPollDuration(25);
ruleEngineConsumerContext.setPackProcessingTimeout(2000);
ruleEngineConsumerContext.setStatsEnabled(false); // true by default
ruleEngineConsumerContext.setPrometheusStatsEnabled(false);
ruleEngineConsumerContext.setTopicDeletionDelayInSec(15);
ruleEngineConsumerContext.setMgmtThreadPoolSize(12);
// Tell the (mock) context factory to return (mock) message pack context
given(packProcessingContextFactory.create(
eq(mainQueue.getName()),
any(TbRuleEngineSubmitStrategy.class),
eq(false)
)).willAnswer(invocation -> {
TbRuleEngineSubmitStrategy realStrategy = invocation.getArgument(1);
when(packCtx.getPendingMap()).thenAnswer(i -> realStrategy.getPendingMap());
when(packCtx.getFailedMap()).thenReturn(new ConcurrentHashMap<>());
return packCtx;
});
// Tell the (mock) context's await() to return 'false' (always timeout) immediately
given(packCtx.await(anyLong(), any(TimeUnit.class))).willReturn(false);
consumerManager = TbRuleEngineQueueConsumerManager.create()
.ctx(ruleEngineConsumerContext)
.queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE, mainQueue))
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.packProcessingContextFactory(packProcessingContextFactory)
.build();
}
@AfterEach
void destroy() {
MoreExecutors.shutdownAndAwaitTermination(scheduler, Duration.ofSeconds(30));
MoreExecutors.shutdownAndAwaitTermination(mgmtExecutor, Duration.ofSeconds(30));
MoreExecutors.shutdownAndAwaitTermination(consumersExecutor, Duration.ofSeconds(30));
}
@Test
void consumerLoopTest_verifyOperationsOrder() throws InterruptedException {
// Create partition
var partition = TopicPartitionInfo.builder()
.tenantId(TenantId.SYS_TENANT_ID)
.topic(mainQueue.getTopic())
.partition(0)
.myPartition(true)
.useInternalPartition(false)
.build();
// Put 10k messages to the queue
for (int i = 0; i < 10_000; i++) {
var tbMsg = TbMsg.newMsg()
.type(TbMsgType.POST_TELEMETRY_REQUEST)
.originator(deviceId)
.data("{\"temperature\":123}")
.metaData(TbMsgMetaData.EMPTY)
.build();
var toRuleEngineMsg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTbMsgProto(TbMsg.toProto(tbMsg))
.addAllRelationTypes(Set.of("Success"))
.build();
storage.put(partition.getFullTopicName(), new TbProtoQueueMsg<>(UUID.randomUUID(), toRuleEngineMsg, new DefaultTbQueueMsgHeaders()));
}
// Count how many polls were made
var totalPolls = new AtomicInteger(0);
var emptyPolls = new AtomicInteger(0);
doAnswer(invocation -> {
totalPolls.incrementAndGet();
@SuppressWarnings("unchecked")
var messages = (List<TbQueueMsg>) invocation.callRealMethod();
if (messages.isEmpty()) {
emptyPolls.incrementAndGet();
}
return messages;
}).when(consumer).poll(mainQueue.getPollInterval());
// Count how many commits were made
var totalCommits = new AtomicInteger(0);
doAnswer(invocation -> {
totalCommits.incrementAndGet();
return invocation.callRealMethod();
}).when(consumer).commit();
// Initialize consumer
consumerManager.init(mainQueue);
// Assign partition to the consumer
consumerManager.update(Set.of(partition));
// Give some time for the consumer to get all messages
await().atMost(Duration.ofSeconds(10L)).until(() -> storage.getLagTotal() == 0);
// Stop consumer
consumerManager.stop();
consumerManager.awaitStop();
// Determine number of non-empty consumer iterations made, since polling does not stop immediately after consuming all messages and may do a few empty polls
int nonEmptyPolls = totalPolls.get() - emptyPolls.get();
// Verify that there is 10 polls and 10 matching commits
// Each poll consumes 1k messages and queue has 10k total, so that means 10k total msgs / 1k msgs per poll = 10 polls
assertThat(nonEmptyPolls).isEqualTo(10).isEqualTo(totalCommits.get());
// Verify that poll-await-commit cycle happened in order with correct await timeout
InOrder inOrder = inOrder(consumer, packCtx);
for (int i = 0; i < nonEmptyPolls; i++) {
inOrder.verify(consumer).poll(mainQueue.getPollInterval());
inOrder.verify(packCtx).await(mainQueue.getPackProcessingTimeout(), TimeUnit.MILLISECONDS);
inOrder.verify(consumer).commit();
}
}
}

2
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java

@ -59,6 +59,7 @@ import org.thingsboard.server.queue.provider.KafkaMonolithQueueFactory;
import org.thingsboard.server.queue.provider.KafkaTbRuleEngineQueueFactory;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
@ -194,6 +195,7 @@ public class TbRuleEngineQueueConsumerManagerTest {
.consumerExecutor(consumersExecutor)
.scheduler(scheduler)
.taskExecutor(mgmtExecutor)
.packProcessingContextFactory(new TbMsgPackProcessingContextFactory.DefaultTbMsgPackProcessingContextFactory())
.build();
}

2
application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineStrategyTest.java

@ -45,6 +45,7 @@ import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask.ConsumerKey;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.service.queue.TbMsgPackProcessingContextFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
@ -196,6 +197,7 @@ public class TbRuleEngineStrategyTest {
var consumerManager = TbRuleEngineQueueConsumerManager.create()
.ctx(ruleEngineConsumerContext)
.queueKey(queueKey)
.packProcessingContextFactory(new TbMsgPackProcessingContextFactory.DefaultTbMsgPackProcessingContextFactory())
.build();
consumerManager.init(queue);

1
application/src/test/java/org/thingsboard/server/service/sms/DefaultSmsServiceTest.java

@ -52,6 +52,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@TestPropertySource(properties = {
"usage.stats.report.enabled=true",
"usage.stats.report.interval=1",
"usage.stats.report.urgent_interval=1"
})
public class DefaultSmsServiceTest extends AbstractControllerTest {
@SpyBean

1
application/src/test/java/org/thingsboard/server/service/stats/DevicesStatisticsTest.java

@ -41,6 +41,7 @@ import static org.awaitility.Awaitility.await;
@TestPropertySource(properties = {
"usage.stats.report.enabled=true",
"usage.stats.report.interval=2",
"usage.stats.report.urgent_interval=1",
"usage.stats.gauge_report_interval=1",
"usage.stats.devices.report_interval=3",
"state.defaultStateCheckIntervalInSec=3",

52
application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java

@ -140,45 +140,61 @@ public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable {
}
private void startDownloading() {
long delay = 0;
// Step 1: state = 1
scheduler.schedule(() -> {
try {
state.set(1);
fireResourceChange(3);
Thread.sleep(100);
state.set(2);
fireResourceChange(3);
} catch (Exception e) {
}
}, 100, TimeUnit.MILLISECONDS);
state.set(1);
fireResourceChange(3);
log.info("Downloading started: state=[{}]", state.get());
}, delay, TimeUnit.MILLISECONDS);
delay += 100; // next step after 100 ms
// Step 2: state = 2
scheduler.schedule(() -> {
state.set(2);
fireResourceChange(3);
log.info("Downloading in progress: state=[{}]", state.get());
}, delay, TimeUnit.MILLISECONDS);
}
private void startUpdating(LwM2mServer identity) {
scheduler.schedule(() -> {
try {
// Update state + result
state.set(3);
fireResourceChange(3);
Thread.sleep(100);
updateResult.set(1);
fireResourceChange(5);
this.pkgName = TITLE;
fireResourceChange(6);
this.pkgVersion = TARGET_FW_VERSION;
fireResourceChange(7);
if (this.leshanClient != null) {
log.info("Stop/reboot LwM2M client {}", this.leshanClient.getEndpoint(identity));
this.leshanClient.stop(false);
log.info("Start after update fw LwM2M client {}", this.leshanClient.getEndpoint(identity));
this.leshanClient.start();
this.pkgName = this.pkgNameDef;
this.pkgVersion = this.pkgVersionDef;
// Delayed reset pkgName/pkgVersion, after reboot + registration
scheduler.schedule(() -> {
this.pkgName = this.pkgNameDef;
fireResourceChange(6);
this.pkgVersion = this.pkgVersionDef;
fireResourceChange(7);
log.info("FW resources updating to new values: pkgName=[{}], pkgVersion=[{}]",
this.pkgName, this.pkgVersion);
}, 15, TimeUnit.SECONDS); // 15 sec — safe timing
}
} catch (Exception e) {
log.error("Error during firmware update", e);
}
}, 100, TimeUnit.MILLISECONDS);
}, 0, TimeUnit.SECONDS); // start immediately, without further delay
}
protected void setLeshanClient(LeshanClient leshanClient) {
this.leshanClient = leshanClient;
}
}

60
application/src/test/java/org/thingsboard/server/transport/lwm2m/client/SwLwM2MDevice.java

@ -33,6 +33,8 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.thingsboard.server.controller.AbstractWebTest.TIMEOUT;
@Slf4j
public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable {
@ -85,10 +87,7 @@ public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable {
log.info("Write on Device resource /{}/{}/{}", getModel().id, getId(), resourceId);
switch (resourceId) {
case 2:
startDownloading();
return WriteResponse.success();
case 3:
case 2, 3:
startDownloading();
return WriteResponse.success();
default:
@ -123,25 +122,34 @@ public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable {
}
private void startDownloading() {
long delay = 0;
// Step 1: start downloading
scheduler.schedule(() -> {
try {
state.set(1);
updateResult.set(1);
fireResourceChange(7);
fireResourceChange(9);
Thread.sleep(100);
state.set(2);
fireResourceChange(7);
Thread.sleep(100);
state.set(3);
fireResourceChange(7);
Thread.sleep(100);
updateResult.set(3);
fireResourceChange(9);
} catch (Exception e) {
}
}, 100, TimeUnit.MILLISECONDS);
state.set(1);
updateResult.set(1);
fireResourceChange(7);
fireResourceChange(9);
}, delay, TimeUnit.MILLISECONDS);
delay += 100;
// Step 2: downloading in progress
scheduler.schedule(() -> {
state.set(2);
fireResourceChange(7);
}, delay, TimeUnit.MILLISECONDS);
delay += 100;
// Step 3: downloading finished
scheduler.schedule(() -> {
state.set(3);
fireResourceChange(7);
updateResult.set(3);
fireResourceChange(9);
}, delay, TimeUnit.MILLISECONDS);
}
private void startUpdating() {
@ -150,7 +158,13 @@ public class SwLwM2MDevice extends BaseInstanceEnabler implements Destroyable {
updateResult.set(2);
fireResourceChange(7);
fireResourceChange(9);
// Optional: delayed log about FW update
scheduler.schedule(() -> {
log.info("FW resources updating to new values: state=[{}], updateResult=[{}]",
state.get(), updateResult.get());
}, 500, TimeUnit.MILLISECONDS);
}, 100, TimeUnit.MILLISECONDS);
}
}

15
application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java

@ -159,14 +159,17 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte
protected List<Device> connectClientWithCorrectAccessTokenWithNDEATHCreatedDevices(int cntDevices, long ts) throws Exception {
List<Device> devices = new ArrayList<>();
clientWithCorrectNodeAccessTokenWithNDEATH();
MetricDataType metricDataType = Int32;
String key = "Node Metric int32";
String keyInt = "Node Metric int32";
int valueDeviceInt32 = 1024;
SparkplugBProto.Payload.Metric metric = createMetric(valueDeviceInt32, ts, key, metricDataType, -1L);
SparkplugBProto.Payload.Metric metricInt = createMetric(valueDeviceInt32, ts, keyInt, Int32, -1L);
String keyStringEmpty = "Node Metric String Empty";
String valueDeviceStringEmpty = "";
SparkplugBProto.Payload.Metric metricStringEmpty = createMetric(valueDeviceStringEmpty, ts, keyStringEmpty, MetricDataType.String, -1L);
SparkplugBProto.Payload.Builder payloadBirthNode = SparkplugBProto.Payload.newBuilder()
.setTimestamp(ts)
.setSeq(getBdSeqNum());
payloadBirthNode.addMetrics(metric);
payloadBirthNode.addMetrics(metricInt);
payloadBirthNode.addMetrics(metricStringEmpty);
payloadBirthNode.setTimestamp(ts);
if (client.isConnected()) {
client.publish(TOPIC_ROOT_SPB_V_1_0 + "/" + groupId + "/" + SparkplugMessageType.NBIRTH.name() + "/" + edgeNode,
@ -174,14 +177,14 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte
}
valueDeviceInt32 = 4024;
metric = createMetric(valueDeviceInt32, ts, metricBirthName_Int32, metricBirthDataType_Int32, -1L);
metricInt = createMetric(valueDeviceInt32, ts, metricBirthName_Int32, metricBirthDataType_Int32, -1L);
for (int i = 0; i < cntDevices; i++) {
SparkplugBProto.Payload.Builder payloadBirthDevice = SparkplugBProto.Payload.newBuilder()
.setTimestamp(ts)
.setSeq(getSeqNum());
String deviceName = deviceId + "_" + i;
payloadBirthDevice.addMetrics(metric);
payloadBirthDevice.addMetrics(metricInt);
if (client.isConnected()) {
client.publish(TOPIC_ROOT_SPB_V_1_0 + "/" + groupId + "/" + SparkplugMessageType.DBIRTH.name() + "/" + edgeNode + "/" + deviceName,
payloadBirthDevice.build().toByteArray(), 0, false);

11
common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java

@ -27,9 +27,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
/**
* @author Andrew Shvayka
*/
public interface AttributesService {
ListenableFuture<Optional<AttributeKvEntry>> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey);
@ -48,7 +45,13 @@ public interface AttributesService {
List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds);
List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
List<String> findAllKeysByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
ListenableFuture<List<String>> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
List<AttributeKvEntry> findLatestByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
ListenableFuture<List<AttributeKvEntry>> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
int removeAllByEntityId(TenantId tenantId, EntityId entityId);

3
common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.entity;
import com.google.common.util.concurrent.FluentFuture;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
@ -51,4 +52,6 @@ public interface EntityService {
PageData<EntityData> findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query);
ListenableFuture<PageData<EntityData>> findEntityDataByQueryAsync(TenantId tenantId, CustomerId customerId, EntityDataQuery query);
}

7
common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java

@ -30,9 +30,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Optional;
/**
* @author Andrew Shvayka
*/
public interface TimeseriesService {
ListenableFuture<List<ReadTsKvQueryResult>> findAllByQueries(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries);
@ -65,6 +62,10 @@ public interface TimeseriesService {
ListenableFuture<List<String>> findAllKeysByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds);
List<TsKvEntry> findLatestByEntityIds(TenantId tenantId, List<EntityId> entityIds);
ListenableFuture<List<TsKvEntry>> findLatestByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds);
void cleanup(long systemTtl);
}

13
common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageRecordKey.java

@ -25,8 +25,8 @@ public enum ApiUsageRecordKey {
RE_EXEC_COUNT(ApiFeature.RE, "ruleEngineExecutionCount", "ruleEngineExecutionLimit", "Rule Engine execution"),
JS_EXEC_COUNT(ApiFeature.JS, "jsExecutionCount", "jsExecutionLimit", "JavaScript execution"),
TBEL_EXEC_COUNT(ApiFeature.TBEL, "tbelExecutionCount", "tbelExecutionLimit", "Tbel execution"),
EMAIL_EXEC_COUNT(ApiFeature.EMAIL, "emailCount", "emailLimit", "email message"),
SMS_EXEC_COUNT(ApiFeature.SMS, "smsCount", "smsLimit", "SMS message"),
EMAIL_EXEC_COUNT(ApiFeature.EMAIL, "emailCount", "emailLimit", "email message", true, true),
SMS_EXEC_COUNT(ApiFeature.SMS, "smsCount", "smsLimit", "SMS message", true, true),
CREATED_ALARMS_COUNT(ApiFeature.ALARM, "createdAlarmsCount", "createdAlarmsLimit", "alarm"),
ACTIVE_DEVICES("activeDevicesCount"),
INACTIVE_DEVICES("inactiveDevicesCount");
@ -50,21 +50,24 @@ public enum ApiUsageRecordKey {
private final String unitLabel;
@Getter
private final boolean counter;
@Getter
private final boolean urgent; // urgent keys are reported at a shorter interval for quicker usage state updates
ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel) {
this(apiFeature, apiCountKey, apiLimitKey, unitLabel, true);
this(apiFeature, apiCountKey, apiLimitKey, unitLabel, true, false);
}
ApiUsageRecordKey(String apiCountKey) {
this(null, apiCountKey, null, null, false);
this(null, apiCountKey, null, null, false, false);
}
ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel, boolean counter) {
ApiUsageRecordKey(ApiFeature apiFeature, String apiCountKey, String apiLimitKey, String unitLabel, boolean counter, boolean urgent) {
this.apiFeature = apiFeature;
this.apiCountKey = apiCountKey;
this.apiLimitKey = apiLimitKey;
this.unitLabel = unitLabel;
this.counter = counter;
this.urgent = urgent;
}
public static ApiUsageRecordKey[] getKeys(ApiFeature feature) {

8
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfiguration.java

@ -22,14 +22,14 @@ public interface ScheduledUpdateSupportedCalculatedFieldConfiguration extends Ca
boolean isScheduledUpdateEnabled();
@PositiveOrZero
int getScheduledUpdateInterval();
Integer getScheduledUpdateInterval();
void setScheduledUpdateInterval(int interval);
void setScheduledUpdateInterval(Integer interval);
default void validate(long minAllowedScheduledUpdateInterval) {
if (getScheduledUpdateInterval() < minAllowedScheduledUpdateInterval) {
throw new IllegalArgumentException("Scheduled update interval is less than configured " +
"minimum allowed interval in tenant profile: " + minAllowedScheduledUpdateInterval);
throw new IllegalArgumentException("Scheduled update interval (" + getScheduledUpdateInterval() +
" seconds) is less than minimum allowed interval in tenant profile: " + minAllowedScheduledUpdateInterval + " seconds");
}
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/aggregation/RelatedEntitiesAggregationCalculatedFieldConfiguration.java

@ -45,7 +45,7 @@ public class RelatedEntitiesAggregationCalculatedFieldConfiguration implements A
private Output output;
private boolean useLatestTs;
private int scheduledUpdateInterval;
private Integer scheduledUpdateInterval;
@Override
public CalculatedFieldType getType() {

5
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfiguration.java

@ -48,7 +48,7 @@ public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCal
private Map<String, ZoneGroupConfiguration> zoneGroups;
private boolean scheduledUpdateEnabled;
private int scheduledUpdateInterval;
private Integer scheduledUpdateInterval;
@NotNull
private Output output;
@ -88,6 +88,9 @@ public class GeofencingCalculatedFieldConfiguration implements ArgumentsBasedCal
@Override
public void validate() {
if (scheduledUpdateEnabled && scheduledUpdateInterval == null) {
throw new IllegalArgumentException("Refresh interval is required when periodic zone group refresh is enabled.");
}
zoneGroups.forEach((key, value) -> value.validate(key));
}

99
common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeysV2.java

@ -0,0 +1,99 @@
/**
* Copyright © 2016-2026 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.query;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.JsonNode;
import io.swagger.v3.oas.annotations.media.ArraySchema;
import io.swagger.v3.oas.annotations.media.Schema;
import org.jspecify.annotations.Nullable;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Schema(
description = """
Contains unique time series and attribute key names discovered from entities matching a query,
optionally including a sample value for each key."""
)
@JsonInclude(JsonInclude.Include.NON_NULL)
public record AvailableEntityKeysV2(
@Schema(
description = "Set of entity types found among the matched entities.",
example = "[\"DEVICE\", \"ASSET\"]",
requiredMode = Schema.RequiredMode.REQUIRED
)
Set<EntityType> entityTypes,
@ArraySchema(
arraySchema = @Schema(
description = """
List of unique time series keys available on the matched entities, sorted alphabetically.
Omitted when timeseries keys were not requested.""",
nullable = true
),
schema = @Schema(implementation = KeyInfo.class)
)
@Nullable List<KeyInfo> timeseries,
@Schema(
description = """
Map of attribute scope to the list of unique attribute keys available on the matched entities.
Only scopes supported by the matched entity types are included.
Omitted when attribute keys were not requested or when none of the requested scopes apply to the matched entity types.""",
nullable = true
)
@Nullable Map<AttributeScope, List<KeyInfo>> attributes
) {
@Schema(description = "Key name with an optional sample value.")
@JsonInclude(JsonInclude.Include.NON_NULL)
public record KeyInfo(
@Schema(
description = "Key name.",
example = "temperature",
requiredMode = Schema.RequiredMode.REQUIRED
)
String key,
@Schema(
description = "Most recent sample value for this key across the matched entities. Omitted when samples were not requested.",
nullable = true
)
@Nullable KeySample sample
) {}
@Schema(description = "Most recent value and its timestamp.")
public record KeySample(
@Schema(
description = "Timestamp in milliseconds since epoch.", example = "1707000000000",
requiredMode = Schema.RequiredMode.REQUIRED
)
long ts,
@Schema(
description = "Sample value.",
example = "23.5",
requiredMode = Schema.RequiredMode.REQUIRED,
implementation = Object.class
)
JsonNode value
) {}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java

@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.tenant.profile;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.Positive;
import jakarta.validation.constraints.PositiveOrZero;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -173,6 +174,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
@Schema(example = "10")
private long maxArgumentsPerCF = 10;
@Schema(example = "10")
@PositiveOrZero
private int minAllowedScheduledUpdateIntervalInSecForCF = 10;
@Builder.Default
@Schema(example = "2")

4
common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/ScheduledUpdateSupportedCalculatedFieldConfigurationTest.java

@ -47,8 +47,8 @@ public class ScheduledUpdateSupportedCalculatedFieldConfigurationTest {
assertThatThrownBy(() -> cfg.validate(minAllowedInterval))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Scheduled update interval is less than configured " +
"minimum allowed interval in tenant profile: " + minAllowedInterval);
.hasMessage("Scheduled update interval (1 seconds) is less than " +
"minimum allowed interval in tenant profile: " + minAllowedInterval + " seconds");
}
}

14
common/data/src/test/java/org/thingsboard/server/common/data/cf/configuration/geofencing/GeofencingCalculatedFieldConfigurationTest.java

@ -28,6 +28,7 @@ import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates.ENTITY_ID_LATITUDE_ARGUMENT_KEY;
@ -103,4 +104,17 @@ public class GeofencingCalculatedFieldConfigurationTest {
assertThat(allowedZonesArgument.getRefEntityKey()).isEqualTo(new ReferencedEntityKey("perimeter", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE));
}
@Test
void validateShouldThrowWhenScheduledUpdateEnabledButIntervalNotSet() {
var cfg = new GeofencingCalculatedFieldConfiguration();
cfg.setEntityCoordinates(mock(EntityCoordinates.class));
cfg.setZoneGroups(Map.of("zone", mock(ZoneGroupConfiguration.class)));
cfg.setScheduledUpdateEnabled(true);
cfg.setScheduledUpdateInterval(null);
assertThatThrownBy(cfg::validate)
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Refresh interval is required when periodic zone group refresh is enabled.");
}
}

8
common/edge-api/src/main/proto/edge.proto

@ -46,12 +46,12 @@ enum EdgeVersion {
V_4_2_0 = 12;
V_4_3_0 = 13;
V_4_2_1_2 = 14;
V_4_2_1_3 = 420;
V_4_2_2 = 4220;
V_4_3_0_1 = 15;
V_4_3_0_2 = 430;
V_4_4_0 = 440;
V_4_3_1 = 4310;
V_4_4_0 = 4400;
V_LATEST = 999;
V_LATEST = 99999;
}
/**

4
common/message/src/test/java/org/thingsboard/server/common/msg/tools/TbRateLimitsTest.java

@ -42,7 +42,7 @@ public class TbRateLimitsTest {
assertThat(rateLimits.tryConsume()).as("new token is available").isFalse();
int expectedRefillTime = (int) (((double) period / capacity) * 1000);
int gap = 500;
int gap = 1000;
for (int i = 0; i < capacity; i++) {
await("token refill for rate limit " + rateLimitConfig)
@ -71,7 +71,7 @@ public class TbRateLimitsTest {
assertThat(rateLimits.tryConsume()).as("new token is available").isFalse();
int expectedRefillTime = period * 1000;
int gap = 500;
int gap = 1000;
await("tokens refill for rate limit " + rateLimitConfig)
.pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS))

4
common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java

@ -156,11 +156,7 @@ public class ProtoConverter {
case BOOLEAN_V:
case LONG_V:
case DOUBLE_V:
break;
case STRING_V:
if (StringUtils.isEmpty(keyValueProto.getStringV())) {
throw new IllegalArgumentException("Value is empty for key: " + key + "!");
}
break;
case JSON_V:
try {

20
common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java

@ -63,8 +63,10 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient {
private boolean enabled;
@Value("${usage.stats.report.enabled_per_customer:false}")
private boolean enabledPerCustomer;
@Value("${usage.stats.report.interval:10}")
@Value("${usage.stats.report.interval:60}")
private int interval;
@Value("${usage.stats.report.urgent_interval:10}")
private int urgentInterval;
@Value("${usage.stats.report.pack_size:1024}")
private int packSize;
@ -83,20 +85,30 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient {
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
stats.put(key, new ConcurrentHashMap<>());
}
Random random = new Random();
scheduler.scheduleWithFixedDelay(() -> {
try {
reportStats();
reportStats(false);
} catch (Exception e) {
log.warn("Failed to report statistics: ", e);
}
}, new Random().nextInt(interval), interval, TimeUnit.SECONDS);
}, random.nextInt(interval), interval, TimeUnit.SECONDS);
scheduler.scheduleWithFixedDelay(() -> {
try {
reportStats(true);
} catch (Exception e) {
log.warn("Failed to report urgent statistics: ", e);
}
}, random.nextInt(urgentInterval), urgentInterval, TimeUnit.SECONDS);
}
}
private void reportStats() {
private void reportStats(boolean urgent) {
ConcurrentMap<ParentEntity, UsageStatsServiceMsg.Builder> report = new ConcurrentHashMap<>();
for (ApiUsageRecordKey key : ApiUsageRecordKey.values()) {
if (key.isUrgent() != urgent) continue;
ConcurrentMap<ReportLevel, AtomicLong> statsForKey = stats.get(key);
statsForKey.forEach((reportLevel, statsValue) -> {
long value = statsValue.get();

6
dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java

@ -55,6 +55,12 @@ public interface AttributesDao {
List<String> findAllKeysByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
ListenableFuture<List<String>> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
List<AttributeKvEntry> findLatestByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
ListenableFuture<List<AttributeKvEntry>> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope);
List<Pair<AttributeScope, String>> removeAllByEntityId(TenantId tenantId, EntityId entityId);
}

20
dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java

@ -45,9 +45,6 @@ import java.util.Optional;
import static org.thingsboard.server.dao.attributes.AttributeUtils.validate;
/**
* @author Andrew Shvayka
*/
@Service
@ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true)
@Primary
@ -92,7 +89,7 @@ public class BaseAttributesService implements AttributesService {
}
@Override
public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
public List<String> findAllKeysByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
if (scope == null) {
return attributesDao.findAllKeysByEntityIds(tenantId, entityIds);
} else {
@ -100,6 +97,21 @@ public class BaseAttributesService implements AttributesService {
}
}
@Override
public ListenableFuture<List<String>> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return attributesDao.findAllKeysByEntityIdsAndScopeAsync(tenantId, entityIds, scope);
}
@Override
public List<AttributeKvEntry> findLatestByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return attributesDao.findLatestByEntityIdsAndScope(tenantId, entityIds, scope);
}
@Override
public ListenableFuture<List<AttributeKvEntry>> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return attributesDao.findLatestByEntityIdsAndScopeAsync(tenantId, entityIds, scope);
}
@Override
public ListenableFuture<AttributesSaveResult> save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) {
validate(entityId, scope);

18
dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java

@ -66,6 +66,7 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate;
@Primary
@Slf4j
public class CachedAttributesService implements AttributesService {
private static final String STATS_NAME = "attributes.cache";
public static final String LOCAL_CACHE_TYPE = "caffeine";
@ -212,7 +213,7 @@ public class CachedAttributesService implements AttributesService {
}
@Override
public List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
public List<String> findAllKeysByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
if (scope == null) {
return attributesDao.findAllKeysByEntityIds(tenantId, entityIds);
} else {
@ -220,6 +221,21 @@ public class CachedAttributesService implements AttributesService {
}
}
@Override
public ListenableFuture<List<String>> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return attributesDao.findAllKeysByEntityIdsAndScopeAsync(tenantId, entityIds, scope);
}
@Override
public List<AttributeKvEntry> findLatestByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return attributesDao.findLatestByEntityIdsAndScope(tenantId, entityIds, scope);
}
@Override
public ListenableFuture<List<AttributeKvEntry>> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return attributesDao.findLatestByEntityIdsAndScopeAsync(tenantId, entityIds, scope);
}
@Override
public ListenableFuture<AttributesSaveResult> save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) {
validate(entityId, scope);

14
dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java

@ -26,18 +26,21 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldFilter;
import org.thingsboard.server.common.data.cf.CalculatedFieldInfo;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.entity.AbstractEntityService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.validator.CalculatedFieldDataValidator;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import java.util.EnumSet;
import java.util.List;
@ -62,6 +65,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements
private final EntityService entityService;
private final CalculatedFieldDao calculatedFieldDao;
private final CalculatedFieldDataValidator calculatedFieldDataValidator;
private final ApiLimitService apiLimitService;
@Override
public CalculatedField save(CalculatedField calculatedField) {
@ -70,6 +74,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements
@Override
public CalculatedField save(CalculatedField calculatedField, boolean doValidate) {
setConfigurationDefaults(calculatedField);
CalculatedField oldCalculatedField = null;
if (doValidate) {
oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId);
@ -79,6 +84,15 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements
return doSave(calculatedField, oldCalculatedField);
}
private void setConfigurationDefaults(CalculatedField calculatedField) {
if (calculatedField.getConfiguration() instanceof RelatedEntitiesAggregationCalculatedFieldConfiguration config
&& config.getScheduledUpdateInterval() == null) {
int minScheduledUpdateInterval = (int) apiLimitService.getLimit(
calculatedField.getTenantId(), DefaultTenantProfileConfiguration::getMinAllowedScheduledUpdateIntervalInSecForCF
);
config.setScheduledUpdateInterval(minScheduledUpdateInterval);
}
}
private CalculatedField doSave(CalculatedField calculatedField, CalculatedField oldCalculatedField) {
try {

86
dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java

@ -16,6 +16,9 @@
package org.thingsboard.server.dao.entity;
import com.google.common.util.concurrent.FluentFuture;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
@ -55,6 +58,7 @@ import org.thingsboard.server.common.msg.edqs.EdqsService;
import org.thingsboard.server.common.stats.EdqsStatsService;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.sql.JpaExecutorService;
import java.util.ArrayList;
import java.util.Collections;
@ -104,6 +108,9 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
@Autowired
private EdqsStatsService edqsStatsService;
@Autowired
private JpaExecutorService jpaExecutorService;
@Override
public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) {
log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query);
@ -142,41 +149,78 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe
EdqsResponse response = processEdqsRequest(tenantId, customerId, request);
result = response.getEntityDataQueryResult();
} else {
if (!isValidForOptimization(query)) {
result = entityQueryDao.findEntityDataByQuery(tenantId, customerId, query);
} else {
// 1 step - find entity data by filter and sort columns
PageData<EntityData> entityDataByQuery = findEntityIdsByFilterAndSorterColumns(tenantId, customerId, query);
if (entityDataByQuery == null || entityDataByQuery.getData().isEmpty()) {
result = entityDataByQuery;
} else {
// 2 step - find entity data by entity ids from the 1st step
List<EntityData> entities = fetchEntityDataByIdsFromInitialQuery(tenantId, customerId, query, entityDataByQuery.getData());
result = new PageData<>(entities, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext());
}
}
result = findEntityDataByQueryInternal(tenantId, customerId, query);
}
edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs);
return result;
}
@Override
public ListenableFuture<PageData<EntityData>> findEntityDataByQueryAsync(TenantId tenantId, CustomerId customerId, EntityDataQuery query) {
log.trace("Executing findEntityDataByQueryAsync, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query);
try {
validateId(tenantId, id -> INCORRECT_TENANT_ID + id);
validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id);
validateEntityDataQuery(query);
} catch (Exception e) {
return Futures.immediateFailedFuture(e);
}
if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) {
EdqsRequest request = EdqsRequest.builder()
.entityDataQuery(query)
.build();
long startNs = System.nanoTime();
return Futures.transform(processEdqsRequestAsync(tenantId, customerId, request), response -> {
edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs);
return response.getEntityDataQueryResult();
}, MoreExecutors.directExecutor());
}
return jpaExecutorService.submit(() -> {
long startNs = System.nanoTime();
PageData<EntityData> result = findEntityDataByQueryInternal(tenantId, customerId, query);
edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs);
return result;
});
}
private PageData<EntityData> findEntityDataByQueryInternal(TenantId tenantId, CustomerId customerId, EntityDataQuery query) {
if (!isValidForOptimization(query)) {
return entityQueryDao.findEntityDataByQuery(tenantId, customerId, query);
}
// 1 step - find entity data by filter and sort columns
PageData<EntityData> entityDataByQuery = findEntityIdsByFilterAndSorterColumns(tenantId, customerId, query);
if (entityDataByQuery == null || entityDataByQuery.getData().isEmpty()) {
return entityDataByQuery;
}
// 2 step - find entity data by entity ids from the 1st step
List<EntityData> entities = fetchEntityDataByIdsFromInitialQuery(tenantId, customerId, query, entityDataByQuery.getData());
return new PageData<>(entities, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext());
}
private boolean validForEdqs(EntityCountQuery query) { // for compatibility with PE
return true;
}
private EdqsResponse processEdqsRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) {
EdqsResponse response;
try {
log.debug("[{}] Sending request to EDQS: {}", tenantId, request);
response = edqsApiService.processRequest(tenantId, customerId, request).get();
return processEdqsRequestAsync(tenantId, customerId, request).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
log.debug("[{}] Received response from EDQS: {}", tenantId, response);
if (response.getError() != null) {
throw new RuntimeException(response.getError());
}
return response;
}
private ListenableFuture<EdqsResponse> processEdqsRequestAsync(TenantId tenantId, CustomerId customerId, EdqsRequest request) {
log.debug("[{}] Sending request to EDQS: {}", tenantId, request);
return Futures.transform(edqsApiService.processRequest(tenantId, customerId, request), response -> {
log.debug("[{}] Received response from EDQS: {}", tenantId, response);
if (response.getError() != null) {
throw new RuntimeException(response.getError());
}
return response;
}, MoreExecutors.directExecutor());
}
@Override

6
dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java

@ -65,6 +65,12 @@ import static org.thingsboard.server.dao.model.ModelConstants.VERSION_COLUMN;
query = SearchTsKvLatestRepository.FIND_ALL_BY_ENTITY_ID_QUERY,
resultSetMapping = "tsKvLatestFindMapping",
resultClass = TsKvLatestEntity.class
),
@NamedNativeQuery(
name = SearchTsKvLatestRepository.FIND_LATEST_BY_ENTITY_IDS,
query = SearchTsKvLatestRepository.FIND_LATEST_BY_ENTITY_IDS_QUERY,
resultSetMapping = "tsKvLatestFindMapping",
resultClass = TsKvLatestEntity.class
)
})
public final class TsKvLatestEntity extends AbstractTsKvEntity {

57
dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java

@ -20,6 +20,14 @@ import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey;
import org.thingsboard.server.dao.model.sql.AttributeKvEntity;
@ -60,6 +68,19 @@ public interface AttributeKvRepository extends JpaRepository<AttributeKvEntity,
List<Integer> findAllKeysByEntityIdsAndAttributeType(@Param("entityIds") List<UUID> entityIds,
@Param("attributeType") int attributeType);
@Query(value = """
SELECT DISTINCT ON (a.attribute_key)
kd.key AS strKey,
a.bool_v AS boolV, a.str_v AS strV, a.long_v AS longV,
a.dbl_v AS dblV, a.json_v AS jsonV,
a.last_update_ts AS lastUpdateTs, a.version AS version
FROM attribute_kv a
INNER JOIN key_dictionary kd ON a.attribute_key = kd.key_id
WHERE a.entity_id IN :entityIds AND a.attribute_type = :attributeType
ORDER BY a.attribute_key, a.last_update_ts DESC""", nativeQuery = true)
List<AttributeKvProjection> findLatestByEntityIdsAndAttributeType(@Param("entityIds") List<UUID> entityIds,
@Param("attributeType") int attributeType);
@Query(value = "SELECT attribute_key, attribute_type, entity_id, bool_v, dbl_v, json_v, last_update_ts, long_v, str_v, version FROM attribute_kv WHERE (entity_id, attribute_type, attribute_key) > " +
"(:entityId, :attributeType, :attributeKey) ORDER BY entity_id, attribute_type, attribute_key LIMIT :batchSize", nativeQuery = true)
List<AttributeKvEntity> findNextBatch(@Param("entityId") UUID entityId,
@ -67,4 +88,40 @@ public interface AttributeKvRepository extends JpaRepository<AttributeKvEntity,
@Param("attributeKey") int attributeKey,
@Param("batchSize") int batchSize);
interface AttributeKvProjection {
String getStrKey();
Boolean getBoolV();
String getStrV();
Long getLongV();
Double getDblV();
String getJsonV();
Long getLastUpdateTs();
Long getVersion();
static AttributeKvEntry toAttributeKvEntry(AttributeKvProjection p) {
KvEntry kvEntry = null;
if (p.getStrV() != null) {
kvEntry = new StringDataEntry(p.getStrKey(), p.getStrV());
} else if (p.getBoolV() != null) {
kvEntry = new BooleanDataEntry(p.getStrKey(), p.getBoolV());
} else if (p.getDblV() != null) {
kvEntry = new DoubleDataEntry(p.getStrKey(), p.getDblV());
} else if (p.getLongV() != null) {
kvEntry = new LongDataEntry(p.getStrKey(), p.getLongV());
} else if (p.getJsonV() != null) {
kvEntry = new JsonDataEntry(p.getStrKey(), p.getJsonV());
}
return new BaseAttributeKvEntry(kvEntry, p.getLastUpdateTs(), p.getVersion());
}
}
}

24
dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java

@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -46,6 +47,7 @@ import org.thingsboard.server.dao.util.SqlDao;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
@ -185,6 +187,28 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl
.toList();
}
@Override
public ListenableFuture<List<String>> findAllKeysByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return service.submit(() -> findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope));
}
@Override
public List<AttributeKvEntry> findLatestByEntityIdsAndScope(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
if (CollectionUtils.isEmpty(entityIds)) {
return Collections.emptyList();
}
var uniqueIds = entityIds.stream().map(EntityId::getId).distinct().toList();
return attributeKvRepository.findLatestByEntityIdsAndAttributeType(uniqueIds, scope.getId())
.stream()
.map(AttributeKvRepository.AttributeKvProjection::toAttributeKvEntry)
.toList();
}
@Override
public ListenableFuture<List<AttributeKvEntry>> findLatestByEntityIdsAndScopeAsync(TenantId tenantId, List<EntityId> entityIds, AttributeScope scope) {
return service.submit(() -> findLatestByEntityIdsAndScope(tenantId, entityIds, scope));
}
@Override
public ListenableFuture<Long> save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute) {
AttributeKvEntity entity = new AttributeKvEntity();

10
dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java

@ -172,4 +172,14 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries
return sqlDao.findAllKeysByEntityIdsAsync(tenantId, entityIds);
}
@Override
public List<TsKvEntry> findLatestByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
return sqlDao.findLatestByEntityIds(tenantId, entityIds);
}
@Override
public ListenableFuture<List<TsKvEntry>> findLatestByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds) {
return sqlDao.findLatestByEntityIdsAsync(tenantId, entityIds);
}
}

17
dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@ -54,6 +55,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao;
import org.thingsboard.server.dao.util.SqlTsLatestAnyDao;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@ -189,6 +191,21 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme
return service.submit(() -> findAllKeysByEntityIds(tenantId, entityIds));
}
@Override
public List<TsKvEntry> findLatestByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
if (CollectionUtils.isEmpty(entityIds)) {
return Collections.emptyList();
}
return DaoUtil.convertDataList(
searchTsKvLatestRepository.findLatestByEntityIds(entityIds.stream().map(EntityId::getId).toList())
);
}
@Override
public ListenableFuture<List<TsKvEntry>> findLatestByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds) {
return service.submit(() -> findLatestByEntityIds(tenantId, entityIds));
}
private ListenableFuture<TsKvLatestRemovingResult> getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query, Long version) {
ListenableFuture<List<TsKvEntry>> future = findNewLatestEntryFuture(tenantId, entityId, query);
return Futures.transformAsync(future, entryList -> {

20
dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java

@ -34,6 +34,20 @@ public class SearchTsKvLatestRepository {
" ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue, ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue, ts_kv_latest.ts AS ts, ts_kv_latest.version AS version FROM ts_kv_latest " +
"INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id WHERE ts_kv_latest.entity_id = cast(:id AS uuid)";
public static final String FIND_LATEST_BY_ENTITY_IDS = "findLatestByEntityIds";
public static final String FIND_LATEST_BY_ENTITY_IDS_QUERY = """
SELECT DISTINCT ON (ts_kv_latest.key)
ts_kv_latest.entity_id AS entityId, ts_kv_latest.key AS key,
key_dictionary.key AS strKey, ts_kv_latest.str_v AS strValue,
ts_kv_latest.bool_v AS boolValue, ts_kv_latest.long_v AS longValue,
ts_kv_latest.dbl_v AS doubleValue, ts_kv_latest.json_v AS jsonValue,
ts_kv_latest.ts AS ts, ts_kv_latest.version AS version
FROM ts_kv_latest
INNER JOIN key_dictionary ON ts_kv_latest.key = key_dictionary.key_id
WHERE ts_kv_latest.entity_id IN (:entityIds)
ORDER BY ts_kv_latest.key, ts_kv_latest.ts DESC""";
@PersistenceContext
private EntityManager entityManager;
@ -43,4 +57,10 @@ public class SearchTsKvLatestRepository {
.getResultList();
}
public List<TsKvLatestEntity> findLatestByEntityIds(List<UUID> entityIds) {
return entityManager.createNamedQuery(FIND_LATEST_BY_ENTITY_IDS, TsKvLatestEntity.class)
.setParameter("entityIds", entityIds)
.getResultList();
}
}

13
dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java

@ -55,9 +55,6 @@ import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.StringUtils.isBlank;
/**
* @author Andrew Shvayka
*/
@Service
@Slf4j
public class BaseTimeseriesService implements TimeseriesService {
@ -161,6 +158,16 @@ public class BaseTimeseriesService implements TimeseriesService {
return timeseriesLatestDao.findAllKeysByEntityIdsAsync(tenantId, entityIds);
}
@Override
public List<TsKvEntry> findLatestByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
return timeseriesLatestDao.findLatestByEntityIds(tenantId, entityIds);
}
@Override
public ListenableFuture<List<TsKvEntry>> findLatestByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds) {
return timeseriesLatestDao.findLatestByEntityIdsAsync(tenantId, entityIds);
}
@Override
public void cleanup(long systemTtl) {
timeseriesDao.cleanup(systemTtl);

10
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java

@ -104,6 +104,16 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes
return Futures.immediateFuture(Collections.emptyList());
}
@Override
public List<TsKvEntry> findLatestByEntityIds(TenantId tenantId, List<EntityId> entityIds) {
return Collections.emptyList();
}
@Override
public ListenableFuture<List<TsKvEntry>> findLatestByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds) {
return Futures.immediateFuture(Collections.emptyList());
}
@Override
public ListenableFuture<Long> saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind());

9
dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java

@ -52,4 +52,13 @@ public interface TimeseriesLatestDao {
ListenableFuture<List<String>> findAllKeysByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds);
/**
* For each unique timeseries key across the given entities, returns the single most recent {@link TsKvEntry}
* (i.e. the entry with the highest timestamp). If the same key exists on multiple entities,
* only the freshest value is kept. Useful for discovering available keys together with a representative sample value.
*/
List<TsKvEntry> findLatestByEntityIds(TenantId tenantId, List<EntityId> entityIds);
ListenableFuture<List<TsKvEntry>> findLatestByEntityIdsAsync(TenantId tenantId, List<EntityId> entityIds);
}

4
dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java

@ -93,11 +93,13 @@ public abstract class AbstractServiceTest {
@Autowired
protected EntityServiceRegistry entityServiceRegistry;
protected Tenant tenant;
protected TenantId tenantId;
@Before
public void beforeAbstractService() {
tenantId = createTenant().getId();
tenant = createTenant();
tenantId = tenant.getId();
}
@After

232
dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.dao.service;
import org.apache.commons.lang3.RandomUtils;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
@ -29,6 +31,10 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey;
import org.thingsboard.server.common.data.cf.configuration.RelationPathQueryDynamicSourceConfiguration;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.TimeSeriesOutput;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggFunction;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggKeyInput;
import org.thingsboard.server.common.data.cf.configuration.aggregation.AggMetric;
import org.thingsboard.server.common.data.cf.configuration.aggregation.RelatedEntitiesAggregationCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.geofencing.EntityCoordinates;
import org.thingsboard.server.common.data.cf.configuration.geofencing.GeofencingCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.geofencing.ZoneGroupConfiguration;
@ -39,6 +45,7 @@ import org.thingsboard.server.common.data.relation.RelationPathLevel;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.exception.DataValidationException;
import java.util.ArrayList;
@ -59,6 +66,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
private DeviceService deviceService;
@Autowired
private TbTenantProfileCache tbTenantProfileCache;
@Autowired
private TenantProfileService tenantProfileService;
@Test
public void testSaveCalculatedField() {
@ -82,8 +91,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName());
assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1);
calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId());
}
@Test
@ -113,11 +120,11 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
int min = tbTenantProfileCache.get(tenantId)
.getDefaultProfileConfiguration()
.getMinAllowedScheduledUpdateIntervalInSecForCF();
int valueFromConfig = min - 10;
// Enable scheduling with an interval below tenant min
cfg.setScheduledUpdateEnabled(true);
cfg.setScheduledUpdateInterval(valueFromConfig);
int invalidInterval = RandomUtils.insecure().randomInt(1, min);
cfg.setScheduledUpdateInterval(invalidInterval);
// Create & save Calculated Field
CalculatedField cf = new CalculatedField();
@ -131,8 +138,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
assertThatThrownBy(() -> calculatedFieldService.save(cf))
.isInstanceOf(DataValidationException.class)
.hasCauseInstanceOf(IllegalArgumentException.class)
.hasMessageStartingWith("Scheduled update interval is less than configured " +
"minimum allowed interval in tenant profile: ");
.hasMessage("Scheduled update interval (" + invalidInterval +
" seconds) is less than minimum allowed interval in tenant profile: " + min + " seconds");
}
@Test
@ -233,8 +240,67 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
int savedInterval = geofencingCalculatedFieldConfiguration.getScheduledUpdateInterval();
assertThat(savedInterval).isEqualTo(valueFromConfig);
}
calculatedFieldService.deleteCalculatedField(tenantId, saved.getId());
@Test
public void testSaveGeofencingCalculatedField_shouldAcceptZeroScheduledUpdateIntervalWhenTenantProfileAllows() {
// GIVEN
var device = createTestDevice();
// Store original value and update tenant profile to allow 0 as min scheduled update interval
TenantProfile tenantProfile = tenantProfileService.findTenantProfileById(tenantId, tenant.getTenantProfileId());
int originalMinScheduledUpdateInterval = tenantProfile.getDefaultProfileConfiguration().getMinAllowedScheduledUpdateIntervalInSecForCF();
tenantProfile.getDefaultProfileConfiguration().setMinAllowedScheduledUpdateIntervalInSecForCF(0);
tenantProfileService.saveTenantProfile(tenantId, tenantProfile);
tbTenantProfileCache.evict(tenantProfile.getId());
try {
// Build a valid Geofencing configuration
var cfg = new GeofencingCalculatedFieldConfiguration();
// Coordinates: TS_LATEST, no dynamic source
var entityCoordinates = new EntityCoordinates("latitude", "longitude");
cfg.setEntityCoordinates(entityCoordinates);
// Zone-group argument (ATTRIBUTE) — make it DYNAMIC so scheduling is enabled
var zoneGroupConfiguration = new ZoneGroupConfiguration("allowed", REPORT_TRANSITION_EVENTS_AND_PRESENCE_STATUS, false);
var dynamicSourceConfiguration = new RelationPathQueryDynamicSourceConfiguration();
dynamicSourceConfiguration.setLevels(List.of(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE)));
zoneGroupConfiguration.setRefDynamicSourceConfiguration(dynamicSourceConfiguration);
cfg.setZoneGroups(Map.of("allowed", zoneGroupConfiguration));
// Enable scheduling with interval = 0
cfg.setScheduledUpdateEnabled(true);
cfg.setScheduledUpdateInterval(0);
// Create Calculated Field
var cf = new CalculatedField();
cf.setTenantId(tenantId);
cf.setEntityId(device.getId());
cf.setType(CalculatedFieldType.GEOFENCING);
cf.setName("GF zero scheduled update interval test");
cf.setConfigurationVersion(0);
cf.setConfiguration(cfg);
var out = new AttributesOutput();
out.setScope(AttributeScope.SERVER_SCOPE);
cfg.setOutput(out);
// WHEN
CalculatedField saved = calculatedFieldService.save(cf);
// THEN
assertThat(saved).isNotNull();
assertThat(saved.getConfiguration()).isInstanceOf(GeofencingCalculatedFieldConfiguration.class);
var savedConfig = (GeofencingCalculatedFieldConfiguration) saved.getConfiguration();
assertThat(savedConfig.getScheduledUpdateInterval()).isEqualTo(0);
} finally {
// Restore original tenant profile value
tenantProfile.getProfileConfiguration().orElseThrow().setMinAllowedScheduledUpdateIntervalInSecForCF(originalMinScheduledUpdateInterval);
tenantProfileService.saveTenantProfile(tenantId, tenantProfile);
tbTenantProfileCache.evict(tenantProfile.getId());
}
}
@Test
@ -254,8 +320,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId());
assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField);
calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId());
}
@Test
@ -267,6 +331,156 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull();
}
@Test
public void testSaveRelatedEntitiesAggregationCF_shouldUseMinScheduledUpdateIntervalFromTenantProfileWhenNotSet() {
// GIVEN
var device = createTestDevice();
var cfg = new RelatedEntitiesAggregationCalculatedFieldConfiguration();
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE));
var argument = new Argument();
argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temp", argument));
var metric = new AggMetric();
metric.setFunction(AggFunction.AVG);
metric.setInput(new AggKeyInput("temp"));
cfg.setMetrics(Map.of("avgTemp", metric));
var output = new TimeSeriesOutput();
output.setName("avgTemperature");
cfg.setOutput(output);
int minDeduplicationInterval = (int) tbTenantProfileCache.get(tenantId)
.getDefaultProfileConfiguration()
.getMinAllowedDeduplicationIntervalInSecForCF();
cfg.setDeduplicationIntervalInSec(minDeduplicationInterval);
// Do NOT set scheduledUpdateInterval - it should default to tenant profile min value
var cf = new CalculatedField();
cf.setTenantId(tenantId);
cf.setEntityId(device.getId());
cf.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION);
cf.setName("Related Entities Aggregation CF - default scheduled interval test");
cf.setConfigurationVersion(0);
cf.setConfiguration(cfg);
// WHEN
CalculatedField saved = calculatedFieldService.save(cf);
// THEN
assertThat(saved).isNotNull();
assertThat(saved.getConfiguration()).isInstanceOf(RelatedEntitiesAggregationCalculatedFieldConfiguration.class);
var savedConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) saved.getConfiguration();
int expectedMinScheduledUpdateInterval = tbTenantProfileCache.get(tenantId)
.getDefaultProfileConfiguration()
.getMinAllowedScheduledUpdateIntervalInSecForCF();
assertThat(savedConfig.getScheduledUpdateInterval()).isEqualTo(expectedMinScheduledUpdateInterval);
}
@Test
public void testSaveRelatedEntitiesAggregationCF_shouldThrowWhenScheduledUpdateIntervalLessThanMinAllowed() {
// GIVEN
var device = createTestDevice();
var cfg = new RelatedEntitiesAggregationCalculatedFieldConfiguration();
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE));
var argument = new Argument();
argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temp", argument));
var metric = new AggMetric();
metric.setFunction(AggFunction.AVG);
metric.setInput(new AggKeyInput("temp"));
cfg.setMetrics(Map.of("avgTemp", metric));
var output = new TimeSeriesOutput();
output.setName("avgTemperature");
cfg.setOutput(output);
int minDeduplicationInterval = (int) tbTenantProfileCache.get(tenantId)
.getDefaultProfileConfiguration()
.getMinAllowedDeduplicationIntervalInSecForCF();
cfg.setDeduplicationIntervalInSec(minDeduplicationInterval);
int minScheduledUpdateInterval = tbTenantProfileCache.get(tenantId)
.getDefaultProfileConfiguration()
.getMinAllowedScheduledUpdateIntervalInSecForCF();
int invalidInterval = RandomUtils.insecure().randomInt(1, minScheduledUpdateInterval);
cfg.setScheduledUpdateInterval(invalidInterval);
var cf = new CalculatedField();
cf.setTenantId(tenantId);
cf.setEntityId(device.getId());
cf.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION);
cf.setName("Related Entities Aggregation CF - invalid scheduled interval test");
cf.setConfigurationVersion(0);
cf.setConfiguration(cfg);
// WHEN-THEN
assertThatThrownBy(() -> calculatedFieldService.save(cf))
.isInstanceOf(DataValidationException.class)
.hasCauseInstanceOf(IllegalArgumentException.class)
.hasMessage("Scheduled update interval (" + invalidInterval +
" seconds) is less than minimum allowed interval in tenant profile: " + minScheduledUpdateInterval + " seconds");
}
@Test
public void testSaveRelatedEntitiesAggregationCF_shouldAcceptValidScheduledUpdateInterval() {
// GIVEN
var device = createTestDevice();
var cfg = new RelatedEntitiesAggregationCalculatedFieldConfiguration();
cfg.setRelation(new RelationPathLevel(EntitySearchDirection.FROM, EntityRelation.CONTAINS_TYPE));
var argument = new Argument();
argument.setRefEntityKey(new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null));
cfg.setArguments(Map.of("temp", argument));
var metric = new AggMetric();
metric.setFunction(AggFunction.AVG);
metric.setInput(new AggKeyInput("temp"));
cfg.setMetrics(Map.of("avgTemp", metric));
var output = new TimeSeriesOutput();
output.setName("avgTemperature");
cfg.setOutput(output);
int minDeduplicationInterval = (int) tbTenantProfileCache.get(tenantId)
.getDefaultProfileConfiguration()
.getMinAllowedDeduplicationIntervalInSecForCF();
cfg.setDeduplicationIntervalInSec(minDeduplicationInterval);
int minScheduledUpdateInterval = tbTenantProfileCache.get(tenantId)
.getDefaultProfileConfiguration()
.getMinAllowedScheduledUpdateIntervalInSecForCF();
int customScheduledUpdateInterval = minScheduledUpdateInterval + 100;
cfg.setScheduledUpdateInterval(customScheduledUpdateInterval);
var cf = new CalculatedField();
cf.setTenantId(tenantId);
cf.setEntityId(device.getId());
cf.setType(CalculatedFieldType.RELATED_ENTITIES_AGGREGATION);
cf.setName("Related Entities Aggregation CF - valid scheduled interval test");
cf.setConfigurationVersion(0);
cf.setConfiguration(cfg);
// WHEN
CalculatedField saved = calculatedFieldService.save(cf);
// THEN
assertThat(saved).isNotNull();
assertThat(saved.getConfiguration()).isInstanceOf(RelatedEntitiesAggregationCalculatedFieldConfiguration.class);
var savedConfig = (RelatedEntitiesAggregationCalculatedFieldConfiguration) saved.getConfiguration();
assertThat(savedConfig.getScheduledUpdateInterval()).isEqualTo(customScheduledUpdateInterval);
}
private CalculatedField saveValidCalculatedField() {
Device device = createTestDevice();
CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId());

103
dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java

@ -23,7 +23,6 @@ import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.AttributeScope;
@ -31,8 +30,12 @@ import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.dao.attributes.AttributesDao;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.service.AbstractServiceTest;
@ -40,6 +43,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.Executors;
@ -48,9 +52,6 @@ import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Andrew Shvayka
*/
@Slf4j
public abstract class BaseAttributesServiceTest extends AbstractServiceTest {
@ -60,9 +61,8 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest {
@Autowired
private AttributesService attributesService;
@Before
public void before() {
}
@Autowired
private AttributesDao attributesDao;
@Test
public void saveAndFetch() throws Exception {
@ -223,7 +223,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest {
saveAttribute(tenantId, deviceId, AttributeScope.SERVER_SCOPE, "key2", "123");
Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
List<String> keys = attributesService.findAllKeysByEntityIds(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE);
List<String> keys = attributesService.findAllKeysByEntityIdsAndScope(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE);
assertThat(keys).containsOnly("key1", "key2");
});
}
@ -241,6 +241,84 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest {
});
}
@Test
public void findLatestByEntityIdsAndScope_returnsOneEntryPerKey() {
var device1 = new DeviceId(UUID.randomUUID());
var device2 = new DeviceId(UUID.randomUUID());
// Both devices have "temperature", device2 has a newer ts
saveAttribute(tenantId, device1, AttributeScope.SERVER_SCOPE, 1000, new DoubleDataEntry("temperature", 20.0));
saveAttribute(tenantId, device2, AttributeScope.SERVER_SCOPE, 2000, new DoubleDataEntry("temperature", 25.0));
// Only device1 has "humidity"
saveAttribute(tenantId, device1, AttributeScope.SERVER_SCOPE, 1500, new LongDataEntry("humidity", 60L));
// Only device2 has "active"
saveAttribute(tenantId, device2, AttributeScope.SERVER_SCOPE, 3000, new BooleanDataEntry("active", true));
List<AttributeKvEntry> results = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device1, device2), AttributeScope.SERVER_SCOPE);
Map<String, AttributeKvEntry> byKey = results.stream().collect(Collectors.toMap(AttributeKvEntry::getKey, e -> e));
Assert.assertEquals(3, byKey.size());
// "temperature" should pick device2's value (ts=2000 > ts=1000)
AttributeKvEntry temp = byKey.get("temperature");
Assert.assertNotNull(temp);
Assert.assertEquals(25.0, temp.getDoubleValue().orElseThrow(), 0.0);
Assert.assertEquals(2000, temp.getLastUpdateTs());
// "humidity" — only device1 has it
AttributeKvEntry humidity = byKey.get("humidity");
Assert.assertNotNull(humidity);
Assert.assertEquals(60L, (long) humidity.getLongValue().orElseThrow());
Assert.assertEquals(1500, humidity.getLastUpdateTs());
// "active" — only device2 has it
AttributeKvEntry active = byKey.get("active");
Assert.assertNotNull(active);
Assert.assertEquals(true, active.getBooleanValue().orElseThrow());
Assert.assertEquals(3000, active.getLastUpdateTs());
}
@Test
public void findLatestByEntityIdsAndScope_emptyList() {
List<AttributeKvEntry> results = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(), AttributeScope.SERVER_SCOPE);
Assert.assertTrue(results.isEmpty());
}
@Test
public void findLatestByEntityIdsAndScope_singleEntity() throws Exception {
var device = new DeviceId(UUID.randomUUID());
saveAttribute(tenantId, device, AttributeScope.SERVER_SCOPE, 1000, new StringDataEntry("key1", "value1"));
saveAttribute(tenantId, device, AttributeScope.SERVER_SCOPE, 2000, new StringDataEntry("key2", "value2"));
// sync
List<AttributeKvEntry> results = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device), AttributeScope.SERVER_SCOPE);
Assert.assertEquals(2, results.size());
Map<String, AttributeKvEntry> byKey = results.stream().collect(Collectors.toMap(AttributeKvEntry::getKey, e -> e));
Assert.assertEquals("value1", byKey.get("key1").getStrValue().orElseThrow());
Assert.assertEquals(1000, byKey.get("key1").getLastUpdateTs());
Assert.assertEquals("value2", byKey.get("key2").getStrValue().orElseThrow());
Assert.assertEquals(2000, byKey.get("key2").getLastUpdateTs());
// async — same result
List<AttributeKvEntry> asyncResults = attributesDao.findLatestByEntityIdsAndScopeAsync(tenantId, List.of(device), AttributeScope.SERVER_SCOPE).get();
Assert.assertEquals(results, asyncResults);
}
@Test
public void findLatestByEntityIdsAndScope_filtersScope() {
var device = new DeviceId(UUID.randomUUID());
saveAttribute(tenantId, device, AttributeScope.SERVER_SCOPE, 1000, new StringDataEntry("serverKey", "sv"));
saveAttribute(tenantId, device, AttributeScope.CLIENT_SCOPE, 1000, new StringDataEntry("clientKey", "cv"));
List<AttributeKvEntry> serverResults = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device), AttributeScope.SERVER_SCOPE);
Assert.assertEquals(1, serverResults.size());
Assert.assertEquals("serverKey", serverResults.get(0).getKey());
List<AttributeKvEntry> clientResults = attributesDao.findLatestByEntityIdsAndScope(tenantId, List.of(device), AttributeScope.CLIENT_SCOPE);
Assert.assertEquals(1, clientResults.size());
Assert.assertEquals("clientKey", clientResults.get(0).getKey());
}
private void testConcurrentFetchAndUpdate(TenantId tenantId, DeviceId deviceId, ListeningExecutorService pool) throws Exception {
var scope = AttributeScope.SERVER_SCOPE;
var key = "TEST";
@ -313,6 +391,15 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest {
}
}
private void saveAttribute(TenantId tenantId, DeviceId deviceId, AttributeScope scope, long ts, KvEntry value) {
try {
attributesService.save(tenantId, deviceId, scope, Collections.singletonList(new BaseAttributeKvEntry(ts, value))).get(10, TimeUnit.SECONDS);
} catch (Exception e) {
log.warn("Failed to save attribute", e.getCause());
throw new RuntimeException(e);
}
}
private void equalsIgnoreVersion(AttributeKvEntry expected, AttributeKvEntry actual) {
Assert.assertEquals(expected.getKey(), actual.getKey());
Assert.assertEquals(expected.getValue(), actual.getValue());

56
dao/src/test/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidatorTest.java

@ -16,29 +16,35 @@
package org.thingsboard.server.dao.service.validator;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.bean.override.mockito.MockitoBean;
import org.springframework.test.context.bean.override.mockito.MockitoSpyBean;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.dao.tenant.TenantProfileDao;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.exception.DataValidationException;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.verify;
@SpringBootTest(classes = TenantProfileDataValidator.class)
class TenantProfileDataValidatorTest {
@MockBean
@MockitoBean
TenantProfileDao tenantProfileDao;
@MockBean
@MockitoBean
TenantProfileService tenantProfileService;
@SpyBean
@MockitoSpyBean
TenantProfileDataValidator validator;
TenantId tenantId = TenantId.fromUUID(UUID.fromString("9ef79cdf-37a8-4119-b682-2e7ed4e018da"));
@Test
@ -53,4 +59,44 @@ class TenantProfileDataValidatorTest {
verify(validator).validateString("Tenant profile name", tenantProfile.getName());
}
@ParameterizedTest
@ValueSource(ints = {-1, -100, Integer.MIN_VALUE})
void minAllowedScheduledUpdateIntervalInSecForCF_shouldRejectNegativeValues(int value) {
// GIVEN
var config = new DefaultTenantProfileConfiguration();
config.setMinAllowedScheduledUpdateIntervalInSecForCF(value);
var tenantProfileData = new TenantProfileData();
tenantProfileData.setConfiguration(config);
var tenantProfile = new TenantProfile();
tenantProfile.setName("Test");
tenantProfile.setProfileData(tenantProfileData);
// WHEN/THEN
assertThatThrownBy(() -> validator.validate(tenantProfile, __ -> TenantId.SYS_TENANT_ID))
.isInstanceOf(DataValidationException.class)
.hasMessageContaining("minAllowedScheduledUpdateIntervalInSecForCF")
.hasMessageContaining("must be greater than or equal to 0");
}
@ParameterizedTest
@ValueSource(ints = {0, 1, 60, Integer.MAX_VALUE})
void minAllowedScheduledUpdateIntervalInSecForCF_shouldAcceptValidValues(int value) {
// GIVEN
var config = new DefaultTenantProfileConfiguration();
config.setMinAllowedScheduledUpdateIntervalInSecForCF(value);
var tenantProfileData = new TenantProfileData();
tenantProfileData.setConfiguration(config);
var tenantProfile = new TenantProfile();
tenantProfile.setName("Test");
tenantProfile.setProfileData(tenantProfileData);
// WHEN/THEN
assertThatCode(() -> validator.validate(tenantProfile, __ -> TenantId.SYS_TENANT_ID))
.doesNotThrowAnyException();
}
}

68
dao/src/test/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDaoTest.java

@ -22,6 +22,9 @@ import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.service.AbstractServiceTest;
@ -30,7 +33,9 @@ import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@ -102,6 +107,69 @@ public class SqlTimeseriesLatestDaoTest extends AbstractServiceTest {
}
}
@Test
public void findLatestByEntityIds_returnsOneEntryPerKey() throws Exception {
DeviceId device1 = new DeviceId(UUID.randomUUID());
DeviceId device2 = new DeviceId(UUID.randomUUID());
// Both devices have "temperature" key, device2 has a newer ts
timeseriesLatestDao.saveLatest(tenantId, device1, new BasicTsKvEntry(1000, new DoubleDataEntry("temperature", 20.0))).get();
timeseriesLatestDao.saveLatest(tenantId, device2, new BasicTsKvEntry(2000, new DoubleDataEntry("temperature", 25.0))).get();
// Only device1 has "humidity"
timeseriesLatestDao.saveLatest(tenantId, device1, new BasicTsKvEntry(1500, new LongDataEntry("humidity", 60L))).get();
// Only device2 has "active"
timeseriesLatestDao.saveLatest(tenantId, device2, new BasicTsKvEntry(3000, new BooleanDataEntry("active", true))).get();
List<TsKvEntry> results = timeseriesLatestDao.findLatestByEntityIds(tenantId, List.of(device1, device2));
Map<String, TsKvEntry> byKey = results.stream().collect(Collectors.toMap(TsKvEntry::getKey, e -> e));
assertEquals(3, byKey.size());
// "temperature" should pick device2's value (ts=2000 > ts=1000)
TsKvEntry temp = byKey.get("temperature");
assertNotNull(temp);
assertEquals(25.0, temp.getDoubleValue().orElseThrow());
assertEquals(2000, temp.getTs());
// "humidity" — only device1 has it
TsKvEntry humidity = byKey.get("humidity");
assertNotNull(humidity);
assertEquals(60L, humidity.getLongValue().orElseThrow());
assertEquals(1500, humidity.getTs());
// "active" — only device2 has it
TsKvEntry active = byKey.get("active");
assertNotNull(active);
assertEquals(true, active.getBooleanValue().orElseThrow());
assertEquals(3000, active.getTs());
}
@Test
public void findLatestByEntityIds_emptyList() {
List<TsKvEntry> results = timeseriesLatestDao.findLatestByEntityIds(tenantId, List.of());
assertTrue(results.isEmpty());
}
@Test
public void findLatestByEntityIds_singleEntity() throws Exception {
DeviceId device = new DeviceId(UUID.randomUUID());
timeseriesLatestDao.saveLatest(tenantId, device, new BasicTsKvEntry(1000, new StringDataEntry("key1", "value1"))).get();
timeseriesLatestDao.saveLatest(tenantId, device, new BasicTsKvEntry(2000, new StringDataEntry("key2", "value2"))).get();
// sync
List<TsKvEntry> results = timeseriesLatestDao.findLatestByEntityIds(tenantId, List.of(device));
assertEquals(2, results.size());
Map<String, TsKvEntry> byKey = results.stream().collect(Collectors.toMap(TsKvEntry::getKey, e -> e));
assertEquals("value1", byKey.get("key1").getStrValue().orElseThrow());
assertEquals(1000, byKey.get("key1").getTs());
assertEquals("value2", byKey.get("key2").getStrValue().orElseThrow());
assertEquals(2000, byKey.get("key2").getTs());
// async — same result
List<TsKvEntry> asyncResults = timeseriesLatestDao.findLatestByEntityIdsAsync(tenantId, List.of(device)).get();
assertEquals(results, asyncResults);
}
private TsKvEntry createEntry(String key, long ts) {
return new BasicTsKvEntry(ts, new StringDataEntry(key, RandomStringUtils.random(10)));
}

2
edqs/src/main/resources/edqs.yml

@ -210,7 +210,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
health:
elasticsearch:
# Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck

4
msa/vc-executor/src/main/resources/tb-vc-executor.yml

@ -215,6 +215,8 @@ usage:
enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
# Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
# Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
# Amount of statistic messages in pack
pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
@ -232,7 +234,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
# Service common properties
service:

63
rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java

@ -94,6 +94,8 @@ import org.thingsboard.server.common.data.asset.AssetSearchQuery;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.audit.AuditLog;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldInfo;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.device.DeviceSearchQuery;
import org.thingsboard.server.common.data.domain.Domain;
import org.thingsboard.server.common.data.domain.DomainInfo;
@ -169,6 +171,7 @@ import org.thingsboard.server.common.data.query.AlarmCountQuery;
import org.thingsboard.server.common.data.query.AlarmData;
import org.thingsboard.server.common.data.query.AlarmDataQuery;
import org.thingsboard.server.common.data.query.AvailableEntityKeys;
import org.thingsboard.server.common.data.query.AvailableEntityKeysV2;
import org.thingsboard.server.common.data.query.EntityCountQuery;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
@ -1895,6 +1898,10 @@ public class RestClient implements Closeable {
}).getBody();
}
/**
* @deprecated Use {@link #findAvailableEntityKeysV2(EntityDataQuery, boolean, boolean, Set, boolean)} instead.
*/
@Deprecated(forRemoval = true)
public AvailableEntityKeys findAvailableEntityKeysByQuery(EntityDataQuery query, boolean includeTimeseries, boolean includeAttributes, AttributeScope scope) {
var uri = UriComponentsBuilder.fromUriString(baseURL)
.path("/api/entitiesQuery/find/keys")
@ -1906,6 +1913,22 @@ public class RestClient implements Closeable {
return restTemplate.exchange(uri, HttpMethod.POST, new HttpEntity<>(query), new ParameterizedTypeReference<AvailableEntityKeys>() {}).getBody();
}
@SneakyThrows(URISyntaxException.class)
public AvailableEntityKeysV2 findAvailableEntityKeysV2(
EntityDataQuery query, boolean includeTimeseries, boolean includeAttributes, Set<AttributeScope> scopes, boolean includeSamples
) {
var builder = new URIBuilder(baseURL).appendPath("/api/v2/entitiesQuery/find/keys")
.addParameter("includeTimeseries", String.valueOf(includeTimeseries))
.addParameter("includeAttributes", String.valueOf(includeAttributes))
.addParameter("includeSamples", String.valueOf(includeSamples));
if (scopes != null) {
for (AttributeScope scope : scopes) {
builder.addParameter("scopes", scope.name());
}
}
return restTemplate.exchange(builder.build(), HttpMethod.POST, new HttpEntity<>(query), new ParameterizedTypeReference<AvailableEntityKeysV2>() {}).getBody();
}
public PageData<AlarmData> findAlarmDataByQuery(AlarmDataQuery query) {
return restTemplate.exchange(
baseURL + "/api/alarmsQuery/find",
@ -4368,6 +4391,46 @@ public class RestClient implements Closeable {
}
@SneakyThrows(URISyntaxException.class)
public PageData<CalculatedFieldInfo> getCalculatedFields(PageLink pageLink,
Set<CalculatedFieldType> types,
EntityType entityType,
Set<UUID> entities,
Set<String> names) {
var urlBuilder = new URIBuilder(baseURL).appendPath("/api/calculatedFields");
urlBuilder.addParameter("pageSize", String.valueOf(pageLink.getPageSize()));
urlBuilder.addParameter("page", String.valueOf(pageLink.getPage()));
if (!isEmpty(pageLink.getTextSearch())) {
urlBuilder.addParameter("textSearch", pageLink.getTextSearch());
}
if (pageLink.getSortOrder() != null) {
urlBuilder.addParameter("sortProperty", pageLink.getSortOrder().getProperty());
urlBuilder.addParameter("sortOrder", pageLink.getSortOrder().getDirection().name());
}
if (!CollectionUtils.isEmpty(types)) {
for (CalculatedFieldType type : types) {
urlBuilder.addParameter("types", type.name());
}
}
if (entityType != null) {
urlBuilder.addParameter("entityType", entityType.name());
}
if (!CollectionUtils.isEmpty(entities)) {
for (UUID entity : entities) {
urlBuilder.addParameter("entities", entity.toString());
}
}
if (!CollectionUtils.isEmpty(names)) {
for (String name : names) {
urlBuilder.addParameter("name", name);
}
}
return restTemplate.exchange(
urlBuilder.build(),
HttpMethod.GET, HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<CalculatedFieldInfo>>() {}).getBody();
}
public void deleteCalculatedField(CalculatedFieldId calculatedFieldId) {
restTemplate.delete(baseURL + "/api/calculatedField/{calculatedFieldId}", calculatedFieldId.getId());
}

4
transport/coap/src/main/resources/tb-coap-transport.yml

@ -421,6 +421,8 @@ usage:
enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
# Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
# Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
# Amount of statistic messages in pack
pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
@ -435,7 +437,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
# Notification system parameters
notification_system:

4
transport/http/src/main/resources/tb-http-transport.yml

@ -370,6 +370,8 @@ usage:
enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
# Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
# Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
# Amount of statistic messages in pack
pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
@ -384,7 +386,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
# Notification system parameters
notification_system:

4
transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml

@ -471,6 +471,8 @@ usage:
enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
# Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
# Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
# Amount of statistic messages in pack
pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
@ -485,7 +487,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
# Notification system parameters
notification_system:

4
transport/mqtt/src/main/resources/tb-mqtt-transport.yml

@ -404,6 +404,8 @@ usage:
enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
# Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
# Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
# Amount of statistic messages in pack
pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
@ -418,7 +420,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
# Notification system parameters
notification_system:

4
transport/snmp/src/main/resources/tb-snmp-transport.yml

@ -359,6 +359,8 @@ usage:
enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}"
# Interval of reporting the statistics. By default, the summarized statistics are sent every 10 seconds
interval: "${USAGE_STATS_REPORT_INTERVAL:60}"
# Reporting interval for urgent keys (e.g. SMS, Email) that require quicker usage state updates
urgent_interval: "${USAGE_STATS_REPORT_URGENT_INTERVAL:10}"
# Amount of statistic messages in pack
pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}"
@ -373,7 +375,7 @@ management:
web:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
include: "${METRICS_ENDPOINTS_EXPOSE:info}"
# Notification system parameters
notification_system:

51
ui-ngx/patches/@iplab+ngx-color-picker+20.0.0.patch

@ -1,46 +1,13 @@
diff --git a/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs b/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs
index a372799..a3d709a 100644
index a372799..f64a6f8 100644
--- a/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs
+++ b/node_modules/@iplab/ngx-color-picker/fesm2022/iplab-ngx-color-picker.mjs
@@ -1129,11 +1129,11 @@ class RgbaComponent {
this.color.set(newColor);
@@ -516,7 +516,7 @@ class Color {
const s = (color.saturation / 100) * (l <= 1 ? l : 2 - l);
const value = (l + s) / 2;
const saturation = (2 * s) / (l + s) || 0;
- return new Hsva(hue, saturation, value, color.alpha);
+ return new Hsva(hue, saturation * 100, value * 100, color.alpha);
}
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: RgbaComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); }
- static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: RgbaComponent, isStandalone: true, selector: "rgba-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getRed().toString()\" (inputChange)=\"onInputChange($event, 'R')\" />\r\n @if (labelVisible()) {\r\n <span>R</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getGreen().toString()\" (inputChange)=\"onInputChange($event, 'G')\" />\r\n @if (labelVisible()) {\r\n <span>G</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getBlue().toString()\" (inputChange)=\"onInputChange($event, 'B')\" />\r\n @if (labelVisible()) {\r\n <span>B</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); }
+ static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: RgbaComponent, isStandalone: true, selector: "rgba-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getRed().toString()\" (inputChange)=\"onInputChange($event, 'R')\" />\r\n @if (labelVisible()) {\r\n <span>R</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getGreen().toString()\" (inputChange)=\"onInputChange($event, 'G')\" />\r\n @if (labelVisible()) {\r\n <span>G</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getBlue().toString()\" (inputChange)=\"onInputChange($event, 'B')\" />\r\n @if (labelVisible()) {\r\n <span>B</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); }
}
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: RgbaComponent, decorators: [{
type: Component,
- args: [{ selector: `rgba-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getRed().toString()\" (inputChange)=\"onInputChange($event, 'R')\" />\r\n @if (labelVisible()) {\r\n <span>R</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getGreen().toString()\" (inputChange)=\"onInputChange($event, 'G')\" />\r\n @if (labelVisible()) {\r\n <span>G</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getBlue().toString()\" (inputChange)=\"onInputChange($event, 'B')\" />\r\n @if (labelVisible()) {\r\n <span>B</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }]
+ args: [{ selector: `rgba-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getRed().toString()\" (inputChange)=\"onInputChange($event, 'R')\" />\r\n @if (labelVisible()) {\r\n <span>R</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getGreen().toString()\" (inputChange)=\"onInputChange($event, 'G')\" />\r\n @if (labelVisible()) {\r\n <span>G</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"255\" [value]=\"value?.getBlue().toString()\" (inputChange)=\"onInputChange($event, 'B')\" />\r\n @if (labelVisible()) {\r\n <span>B</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }]
}] });
class HslaComponent {
@@ -1155,11 +1155,11 @@ class HslaComponent {
this.color.set(newColor);
}
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HslaComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); }
- static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HslaComponent, isStandalone: true, selector: "hsla-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"360\" [value]=\"value?.getHue().toString()\" (inputChange)=\"onInputChange($event, 'H')\" />\r\n @if (labelVisible()) {\r\n <span>H</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getSaturation() + '%'\" (inputChange)=\"onInputChange($event, 'S')\" />\r\n @if (labelVisible()) {\r\n <span>S</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getLightness() + '%'\" (inputChange)=\"onInputChange($event, 'L')\" />\r\n @if (labelVisible()) {\r\n <span>L</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); }
+ static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HslaComponent, isStandalone: true, selector: "hsla-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, isAlphaVisible: { classPropertyName: "isAlphaVisible", publicName: "alpha", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"360\" [value]=\"value?.getHue().toString()\" (inputChange)=\"onInputChange($event, 'H')\" />\r\n @if (labelVisible()) {\r\n <span>H</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getSaturation() + '%'\" (inputChange)=\"onInputChange($event, 'S')\" />\r\n @if (labelVisible()) {\r\n <span>S</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getLightness() + '%'\" (inputChange)=\"onInputChange($event, 'L')\" />\r\n @if (labelVisible()) {\r\n <span>L</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], dependencies: [{ kind: "directive", type: ColorPickerInputDirective, selector: "[inputChange]", inputs: ["min", "max"], outputs: ["inputChange"] }], changeDetection: i0.ChangeDetectionStrategy.OnPush }); }
}
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HslaComponent, decorators: [{
type: Component,
- args: [{ selector: `hsla-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"360\" [value]=\"value?.getHue().toString()\" (inputChange)=\"onInputChange($event, 'H')\" />\r\n @if (labelVisible()) {\r\n <span>H</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getSaturation() + '%'\" (inputChange)=\"onInputChange($event, 'S')\" />\r\n @if (labelVisible()) {\r\n <span>S</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getLightness() + '%'\" (inputChange)=\"onInputChange($event, 'L')\" />\r\n @if (labelVisible()) {\r\n <span>L</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"text\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }]
+ args: [{ selector: `hsla-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, imports: [ColorPickerInputDirective], template: "<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"360\" [value]=\"value?.getHue().toString()\" (inputChange)=\"onInputChange($event, 'H')\" />\r\n @if (labelVisible()) {\r\n <span>H</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getSaturation() + '%'\" (inputChange)=\"onInputChange($event, 'S')\" />\r\n @if (labelVisible()) {\r\n <span>S</span>\r\n }\r\n</div>\r\n<div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]*\" min=\"0\" max=\"100\" [value]=\"value?.getLightness() + '%'\" (inputChange)=\"onInputChange($event, 'L')\" />\r\n @if (labelVisible()) {\r\n <span>L</span>\r\n }\r\n</div>\r\n@if (isAlphaVisible()) {\r\n <div class=\"column\">\r\n <input type=\"number\" pattern=\"[0-9]+([\\.,][0-9]{1,2})?\" min=\"0\" max=\"1\" [value]=\"value?.getAlpha(true).toString()\" (inputChange)=\"onInputChange($event, 'A')\" />\r\n @if (labelVisible()) {\r\n <span>A</span>\r\n }\r\n </div>\r\n}", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }]
}] });
class HexComponent {
@@ -1190,11 +1190,11 @@ class HexComponent {
}
}
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HexComponent, deps: [], target: i0.ɵɵFactoryTarget.Component }); }
- static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HexComponent, isStandalone: true, selector: "hex-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, prefixValue: { classPropertyName: "prefixValue", publicName: "prefix", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "<div class=\"column\">\r\n <input #elRef type=\"text\" [value]=\"value\" (keyup)=\"onInputChange($event, elRef.value)\" />\r\n @if (labelVisible()) {\r\n <span>HEX</span>\r\n }\r\n</div>", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], changeDetection: i0.ChangeDetectionStrategy.OnPush }); }
+ static { this.ɵcmp = i0.ɵɵngDeclareComponent({ minVersion: "17.0.0", version: "20.0.4", type: HexComponent, isStandalone: true, selector: "hex-input-component", inputs: { color: { classPropertyName: "color", publicName: "color", isSignal: true, isRequired: true, transformFunction: null }, labelVisible: { classPropertyName: "labelVisible", publicName: "label", isSignal: true, isRequired: false, transformFunction: null }, prefixValue: { classPropertyName: "prefixValue", publicName: "prefix", isSignal: true, isRequired: false, transformFunction: null } }, outputs: { color: "colorChange" }, ngImport: i0, template: "<div class=\"column\">\r\n <input #elRef type=\"number\" [value]=\"value\" (keyup)=\"onInputChange($event, elRef.value)\" />\r\n @if (labelVisible()) {\r\n <span>HEX</span>\r\n }\r\n</div>", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n", ""], changeDetection: i0.ChangeDetectionStrategy.OnPush }); }
}
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.0.4", ngImport: i0, type: HexComponent, decorators: [{
type: Component,
- args: [{ selector: `hex-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, standalone: true, template: "<div class=\"column\">\r\n <input #elRef type=\"text\" [value]=\"value\" (keyup)=\"onInputChange($event, elRef.value)\" />\r\n @if (labelVisible()) {\r\n <span>HEX</span>\r\n }\r\n</div>", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }]
+ args: [{ selector: `hex-input-component`, changeDetection: ChangeDetectionStrategy.OnPush, standalone: true, template: "<div class=\"column\">\r\n <input #elRef type=\"number\" [value]=\"value\" (keyup)=\"onInputChange($event, elRef.value)\" />\r\n @if (labelVisible()) {\r\n <span>HEX</span>\r\n }\r\n</div>", styles: [":host,:host ::ng-deep *{padding:0;margin:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}\n", ":host{display:table;width:100%;text-align:center;color:#b4b4b4;font-size:11px}.column{display:table-cell;padding:0 2px}input{width:100%;border:1px solid rgb(218,218,218);color:#272727;text-align:center;font-size:12px;-webkit-appearance:none;border-radius:0;margin:0 0 6px;height:26px;outline:none}\n"] }]
}] });
const OpacityAnimation = trigger('opacityAnimation', [
rgbaToHsva(color) {
const red = color.red / 255;

31
ui-ngx/patches/ngx-hm-carousel+19.0.0.patch

@ -0,0 +1,31 @@
diff --git a/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs b/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs
index 117f782..70e49a7 100644
--- a/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs
+++ b/node_modules/ngx-hm-carousel/fesm2022/ngx-hm-carousel.mjs
@@ -1,5 +1,5 @@
import * as i0 from '@angular/core';
-import { Directive, inject, ViewContainerRef, TemplateRef, input, PLATFORM_ID, DestroyRef, Renderer2, NgZone, ChangeDetectorRef, viewChild, ElementRef, contentChildren, contentChild, computed, signal, effect, forwardRef, Component, ChangeDetectionStrategy } from '@angular/core';
+import { Directive, inject, ViewContainerRef, TemplateRef, input, PLATFORM_ID, DestroyRef, Renderer2, NgZone, ChangeDetectorRef, viewChild, ElementRef, contentChildren, contentChild, computed, signal, effect, forwardRef, Component, ChangeDetectionStrategy, afterNextRender } from '@angular/core';
import { DOCUMENT, isPlatformBrowser, NgTemplateOutlet, AsyncPipe } from '@angular/common';
import { toObservable, takeUntilDestroyed } from '@angular/core/rxjs-interop';
import { NG_VALUE_ACCESSOR } from '@angular/forms';
@@ -340,7 +340,7 @@ class NgxHmCarouselComponent {
});
});
});
- const effectRef = effect(() => {
+ afterNextRender(() => {
this.rootElm = this.container().nativeElement;
this.containerElm = this.rootElm.children[0];
this.init();
@@ -365,10 +365,6 @@ class NgxHmCarouselComponent {
])
.pipe(takeUntilDestroyed(this._destroyRef))
.subscribe();
- // only exec once
- effectRef.destroy();
- }, {
- allowSignalWrites: true,
});
}
ngOnDestroy() {

4
ui-ngx/src/app/modules/home/components/alarm-rules/alarm-rules-table-config.ts

@ -106,6 +106,8 @@ export class AlarmRulesTableConfig extends EntityTableConfig<AlarmRuleTableEntit
this.entityComponent = AlarmRulesComponent;
this.entityTabsComponent = AlarmRulesTabsComponent;
this.rowPointer = true;
} else {
this.addAsTextButton = false;
}
this.tableTitle = this.pageMode ? '' : this.translate.instant('alarm-rule.alarm-rules');
this.detailsPanelEnabled = this.pageMode;
@ -117,7 +119,7 @@ export class AlarmRulesTableConfig extends EntityTableConfig<AlarmRuleTableEntit
type: 'alarm-rule.alarm-rule',
typePlural: 'alarm-rule.alarm-rules',
list: 'alarm-rule.list',
add: 'action.add',
add: 'alarm-rule.add',
details: 'alarm-rule.details',
noEntities: 'alarm-rule.no-found',
search: 'action.search',

1
ui-ngx/src/app/modules/home/components/api-key/api-keys-table-config.ts

@ -59,7 +59,6 @@ export class ApiKeysTableConfig extends EntityTableConfig<ApiKeyInfo> {
this.entityType = EntityType.API_KEY;
this.detailsPanelEnabled = false;
this.addAsTextButton = true;
this.pageMode = false;
this.entityTranslations = entityTypeTranslations.get(EntityType.API_KEY);

2
ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html

@ -142,7 +142,7 @@
<ng-container matColumnDef="select" sticky>
<mat-header-cell *matHeaderCellDef style="width: 40px;">
<mat-checkbox (change)="$event ? dataSource.masterToggle() : null"
[checked]="dataSource.selection.hasValue() && (dataSource.isAllSelected() | async)"
[(ngModel)]="selectAllModel"
[indeterminate]="dataSource.selection.hasValue() && !(dataSource.isAllSelected() | async)">
</mat-checkbox>
</mat-header-cell>

12
ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts

@ -40,7 +40,7 @@ import { MatDialog } from '@angular/material/dialog';
import { DialogService } from '@core/services/dialog.service';
import { Direction, SortOrder } from '@shared/models/page/sort-order';
import { forkJoin, merge, Observable, Subject } from 'rxjs';
import { debounceTime, distinctUntilChanged, takeUntil } from 'rxjs/operators';
import { debounceTime, distinctUntilChanged, take, takeUntil } from 'rxjs/operators';
import { EntityId } from '@shared/models/id/entity-id';
import {
AttributeData,
@ -187,6 +187,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI
textSearch = this.fb.control('', {nonNullable: true});
private destroy$ = new Subject<void>();
selectAllModel: boolean = false;
constructor(protected store: Store<AppState>,
private attributeService: AttributeService,
@ -223,6 +224,14 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI
});
});
this.widgetResize$.observe(this.elementRef.nativeElement);
this.dataSource.selection.changed.pipe(
takeUntil(this.destroy$)
).subscribe(() => {
this.dataSource.isAllSelected().pipe(take(1)).subscribe(allSelected => {
this.selectAllModel = allSelected;
});
});
}
ngOnDestroy() {
@ -237,6 +246,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI
this.attributeScope = attributeScope;
this.mode = 'default';
this.paginator.pageIndex = 0;
this.selectAllModel = false;
this.updateData(true);
}

2
ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts

@ -110,6 +110,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig<CalculatedFie
this.entityComponent = CalculatedFieldComponent;
this.entityTabsComponent = CalculatedFieldsTabsComponent;
this.rowPointer = true;
} else {
this.addAsTextButton = false;
}
this.tableTitle = this.translate.instant('entity.type-calculated-fields');
this.detailsPanelEnabled = this.pageMode;

104
ui-ngx/src/app/modules/home/components/entity/entities-table.component.html

@ -45,27 +45,19 @@
asButton strokedButton historyOnly [forAllTimeEnabled]="entitiesTableConfig.forAllTimeEnabled"></tb-timewindow>
</div>
<span class="flex-1"></span>
<div [class.!hidden]="!addEnabled()">
<ng-container *ngIf="!entitiesTableConfig.addActionDescriptors.length; else addActions">
<button *ngIf="!entitiesTableConfig.addAsTextButton"
mat-icon-button
<div [class.!hidden]="!addEnabled() || entitiesTableConfig.addAsTextButton">
<ng-container *ngIf="!entitiesTableConfig.addActionDescriptors.length; else iconCustom">
<button mat-icon-button
[disabled]="isLoading$ | async"
(click)="addEntity($event)"
matTooltip="{{ translations.add | translate }}"
matTooltipPosition="above">
<mat-icon>add</mat-icon>
</button>
<button *ngIf="entitiesTableConfig.addAsTextButton"
mat-stroked-button color="primary"
[disabled]="isLoading$ | async"
(click)="addEntity($event)">
<mat-icon>add</mat-icon>
{{ translations.add | translate }}
</button>
</ng-container>
<ng-template #addActions>
<ng-container *ngIf="entitiesTableConfig.addActionDescriptors.length === 1; else addActionsMenu">
<button *ngIf="!entitiesTableConfig.addAsTextButton && entitiesTableConfig.addActionDescriptors[0].isEnabled()"
<ng-template #iconCustom>
<ng-container *ngIf="entitiesTableConfig.addActionDescriptors.length === 1; else iconMenu">
<button *ngIf="entitiesTableConfig.addActionDescriptors[0].isEnabled()"
mat-icon-button
[disabled]="isLoading$ | async"
(click)="entitiesTableConfig.addActionDescriptors[0].onAction($event)"
@ -73,30 +65,14 @@
matTooltipPosition="above">
<tb-icon>{{entitiesTableConfig.addActionDescriptors[0].icon}}</tb-icon>
</button>
<button *ngIf="entitiesTableConfig.addAsTextButton && entitiesTableConfig.addActionDescriptors[0].isEnabled()"
mat-stroked-button color="primary"
[disabled]="isLoading$ | async"
(click)="entitiesTableConfig.addActionDescriptors[0].onAction($event)">
<tb-icon>{{entitiesTableConfig.addActionDescriptors[0].icon}}</tb-icon>
{{ entitiesTableConfig.addActionDescriptors[0].name }}
</button>
</ng-container>
<ng-template #addActionsMenu>
<ng-template #iconMenu>
<button mat-icon-button [disabled]="isLoading$ | async"
matTooltip="{{ translations.add | translate }}"
matTooltipPosition="above"
[matMenuTriggerFor]="addActionsMenu">
<mat-icon>add</mat-icon>
</button>
<mat-menu #addActionsMenu="matMenu" xPosition="before">
<button mat-menu-item *ngFor="let actionDescriptor of entitiesTableConfig.addActionDescriptors"
[disabled]="isLoading$ | async"
[class.!hidden]="!actionDescriptor.isEnabled()"
(click)="actionDescriptor.onAction($event)">
<tb-icon matMenuItemIcon>{{actionDescriptor.icon}}</tb-icon>
<span>{{ actionDescriptor.name }}</span>
</button>
</mat-menu>
</ng-template>
</ng-template>
</div>
@ -118,6 +94,72 @@
matTooltipPosition="above">
<mat-icon>search</mat-icon>
</button>
<div [class.!hidden]="!addEnabled() || !entitiesTableConfig.addAsTextButton" class="pl-3" style="--mat-fab-small-container-elevation-shadow: none;">
<ng-container *ngIf="!entitiesTableConfig.addActionDescriptors.length; else textActions">
<button class="lt-sm:!hidden"
mat-flat-button color="primary"
[disabled]="isLoading$ | async"
(click)="addEntity($event)">
<mat-icon>add</mat-icon>
{{ translations.add | translate }}
</button>
<button class="gt-xs:!hidden"
mat-mini-fab color="primary"
[disabled]="isLoading$ | async"
(click)="addEntity($event)"
matTooltip="{{ translations.add | translate }}"
matTooltipPosition="above">
<mat-icon>add</mat-icon>
</button>
</ng-container>
<ng-template #textActions>
<ng-container *ngIf="entitiesTableConfig.addActionDescriptors.length === 1; else textMenu">
<ng-container *ngIf="entitiesTableConfig.addActionDescriptors[0].isEnabled()">
<button class="lt-sm:!hidden"
mat-flat-button color="primary"
[disabled]="isLoading$ | async"
(click)="entitiesTableConfig.addActionDescriptors[0].onAction($event)">
<tb-icon matButtonIcon>{{entitiesTableConfig.addActionDescriptors[0].icon}}</tb-icon>
{{ entitiesTableConfig.addActionDescriptors[0].name }}
</button>
<button class="gt-xs:!hidden"
mat-mini-fab color="primary"
[disabled]="isLoading$ | async"
(click)="entitiesTableConfig.addActionDescriptors[0].onAction($event)"
matTooltip="{{ entitiesTableConfig.addActionDescriptors[0].name }}"
matTooltipPosition="above">
<mat-icon>{{entitiesTableConfig.addActionDescriptors[0].icon}}</mat-icon>
</button>
</ng-container>
</ng-container>
<ng-template #textMenu>
<button class="lt-sm:!hidden"
mat-flat-button color="primary"
[disabled]="isLoading$ | async"
[matMenuTriggerFor]="addActionsMenu">
<mat-icon>add</mat-icon>
{{ translations.add | translate }}
</button>
<button class="gt-xs:!hidden"
mat-mini-fab color="primary"
[disabled]="isLoading$ | async"
[matMenuTriggerFor]="addActionsMenu"
matTooltip="{{ translations.add | translate }}"
matTooltipPosition="above">
<mat-icon>add</mat-icon>
</button>
</ng-template>
</ng-template>
</div>
<mat-menu #addActionsMenu="matMenu" xPosition="before">
<button mat-menu-item *ngFor="let actionDescriptor of entitiesTableConfig.addActionDescriptors"
[disabled]="isLoading$ | async"
[class.!hidden]="!actionDescriptor.isEnabled()"
(click)="actionDescriptor.onAction($event)">
<tb-icon matMenuItemIcon>{{actionDescriptor.icon}}</tb-icon>
<span>{{ actionDescriptor.name }}</span>
</button>
</mat-menu>
</div>
</mat-toolbar>
<mat-toolbar class="mat-mdc-table-toolbar" [class.!hidden]="!textSearchMode || !dataSource.selection.isEmpty()">

2
ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.scss

@ -43,7 +43,7 @@
}
}
> .mat-expansion-panel-content {
.mat-expansion-panel-content {
> .mat-expansion-panel-body {
padding: 0;
}

18
ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html

@ -32,7 +32,8 @@
</tb-branch-autocomplete>
</div>
<span class="flex-1"></span>
<div class="flex flex-row items-center justify-center xs:flex-col xs:items-end xs:justify-center">
<div class="flex flex-row items-center justify-center xs:flex-col xs:items-end xs:justify-center"
[class.xs:flex-col-reverse]="!singleEntityMode">
<button *ngIf="singleEntityMode" mat-stroked-button color="primary"
#createVersionButton
[disabled]="(isLoading$ | async) || (isReadOnly | async)"
@ -40,13 +41,6 @@
<mat-icon>update</mat-icon>
{{'version-control.create-version' | translate }}
</button>
<button *ngIf="!singleEntityMode" mat-stroked-button color="primary"
#complexCreateVersionButton
[disabled]="(isLoading$ | async) || (isReadOnly | async)"
(click)="toggleComplexCreateVersion($event, complexCreateVersionButton)">
<mat-icon>update</mat-icon>
{{'version-control.create-entities-version' | translate }}
</button>
<div class="flex flex-row">
<button mat-icon-button [disabled]="isLoading$ | async" (click)="updateData()"
matTooltip="{{ 'action.refresh' | translate }}"
@ -61,6 +55,14 @@
<mat-icon>search</mat-icon>
</button>
</div>
<button *ngIf="!singleEntityMode" mat-flat-button color="primary"
class="gt-xs:ml-3"
#complexCreateVersionButton
[disabled]="(isLoading$ | async) || (isReadOnly | async)"
(click)="toggleComplexCreateVersion($event, complexCreateVersionButton)">
<mat-icon>update</mat-icon>
{{'version-control.create-entities-version' | translate }}
</button>
</div>
</div>
</mat-toolbar>

12
ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.ts

@ -45,7 +45,10 @@ import {
barChartWithLabelsDefaultSettings,
BarChartWithLabelsWidgetSettings
} from '@home/components/widget/lib/chart/bar-chart-with-labels-widget.models';
import { TimeSeriesChartType } from '@home/components/widget/lib/chart/time-series-chart.models';
import {
TimeSeriesChartType,
updateLatestDataKeys
} from '@home/components/widget/lib/chart/time-series-chart.models';
import { getSourceTbUnitSymbol } from '@shared/models/unit.models';
@Component({
@ -76,7 +79,7 @@ export class BarChartWithLabelsBasicConfigComponent extends BasicWidgetConfigCom
tooltipDatePreviewFn = this._tooltipDatePreviewFn.bind(this);
predefinedValues = widgetTitleAutocompleteValues;
constructor(protected store: Store<AppState>,
protected widgetConfigComponent: WidgetConfigComponent,
private $injector: Injector,
@ -167,6 +170,11 @@ export class BarChartWithLabelsBasicConfigComponent extends BasicWidgetConfigCom
});
}
protected onConfigChanged(widgetConfig: WidgetConfigComponentData) {
updateLatestDataKeys([widgetConfig.config.settings.yAxis], this.datasource, this.callbacks);
super.onConfigChanged(widgetConfig);
}
protected prepareOutputConfig(config: any): WidgetConfigComponentData {
setTimewindowConfig(this.widgetConfig.config, config.timewindowConfig);
this.widgetConfig.config.datasources = config.datasources;

7
ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts

@ -47,7 +47,7 @@ import {
} from '@home/components/widget/lib/chart/range-chart-widget.models';
import {
lineSeriesStepTypes,
lineSeriesStepTypeTranslations
lineSeriesStepTypeTranslations, updateLatestDataKeys
} from '@home/components/widget/lib/chart/time-series-chart.models';
import {
chartLabelPositions,
@ -289,6 +289,11 @@ export class RangeChartBasicConfigComponent extends BasicWidgetConfigComponent {
return this.widgetConfig;
}
protected onConfigChanged(widgetConfig: WidgetConfigComponentData) {
updateLatestDataKeys([widgetConfig.config.settings.yAxis], this.datasource, this.callbacks);
super.onConfigChanged(widgetConfig);
}
protected validatorTriggers(): string[] {
return ['showTitle', 'showIcon', 'showRangeThresholds', 'fillArea', 'showLine',
'step', 'showPointLabel', 'enablePointLabelBackground', 'showLegend', 'showTooltip', 'tooltipShowDate'];

6
ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.component.ts

@ -134,6 +134,12 @@ export class BarChartWithLabelsWidgetComponent implements OnInit, OnDestroy, Aft
}
}
public onLatestDataUpdated() {
if (this.timeSeriesChart) {
this.timeSeriesChart.latestUpdated();
}
}
public onLegendKeyEnter(key: DataKey) {
this.timeSeriesChart.keyEnter(key);
}

6
ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.component.ts

@ -162,6 +162,12 @@ export class RangeChartWidgetComponent implements OnInit, OnDestroy, AfterViewIn
}
}
public onLatestDataUpdated() {
if (this.timeSeriesChart) {
this.timeSeriesChart.latestUpdated();
}
}
public toggleRangeItem(item: RangeItem) {
item.enabled = !item.enabled;
this.timeSeriesChart.toggleVisualMapRange(item.index);

100
ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts

@ -99,6 +99,8 @@ import {
TimeSeriesChartTooltipWidgetSettings
} from '@home/components/widget/lib/chart/time-series-chart-tooltip.models';
import { TbUnit, TbUnitConverter } from '@shared/models/unit.models';
import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
import { DataKeysCallbacks } from '@home/components/widget/lib/settings/common/key/data-keys.component.models';
type TimeSeriesChartDataEntry = [number, any, number, number];
@ -1495,3 +1497,101 @@ const createSeriesLabelOption = (item: TimeSeriesChartDataItem, show: boolean,
}
return labelOption;
};
export const checkLatestDataKeys = (yAxes: TimeSeriesChartYAxes, datasource: Datasource): TimeSeriesChartYAxes => {
const latestKeys = datasource?.latestDataKeys || [];
const result: TimeSeriesChartYAxes = {};
for (const [id, axis] of Object.entries(yAxes)) {
axis.min = normalizeAxisLimit(axis.min);
axis.max = normalizeAxisLimit(axis.max);
const minCfg = axis.min;
const maxCfg = axis.max;
const minValid = !!minCfg && (
minCfg.type !== ValueSourceType.latestKey ||
latestKeys.some(k => isYAxisKey(k, minCfg))
);
const maxValid = !!maxCfg && (
maxCfg.type !== ValueSourceType.latestKey ||
latestKeys.some(k => isYAxisKey(k, maxCfg))
);
if (minValid && maxValid) {
result[id] = axis;
}
}
return result;
}
export const updateLatestDataKeys = (yAxes: TimeSeriesChartYAxisSettings[], datasource: Datasource, dataKeyCallbacks: DataKeysCallbacks)=> {
if (datasource) {
let latestKeys = datasource.latestDataKeys;
if (!latestKeys) {
latestKeys = [];
datasource.latestDataKeys = latestKeys;
}
const existingYAxisKeys = latestKeys.filter(k => k.settings?.__yAxisMinKey === true || k.settings?.__yAxisMaxKey === true);
const foundYAxisKeys: DataKey[] = [];
for(const yAxis of yAxes) {
const min = yAxis.min as ValueSourceConfig;
const max = yAxis.max as ValueSourceConfig;
if (min && min.type === ValueSourceType.latestKey) {
const found = existingYAxisKeys.find(k => isYAxisKey(k, min));
if (!found) {
const newKey = dataKeyCallbacks.generateDataKey(min.latestKey, min.latestKeyType,
null, true, null);
newKey.settings.__yAxisMinKey = true;
latestKeys.push(newKey);
} else if (foundYAxisKeys.indexOf(found) === -1) {
foundYAxisKeys.push(found);
}
}
if (max && max.type === ValueSourceType.latestKey) {
const found = existingYAxisKeys.find(k => isYAxisKey(k, max));
if (!found) {
const newKey = dataKeyCallbacks.generateDataKey(max.latestKey, max.latestKeyType,
null, true, null);
newKey.settings.__yAxisMaxKey = true;
latestKeys.push(newKey);
} else if (foundYAxisKeys.indexOf(found) === -1) {
foundYAxisKeys.push(found);
}
}
}
const toRemove = existingYAxisKeys.filter(k => foundYAxisKeys.indexOf(k) === -1);
for (const key of toRemove) {
const index = latestKeys.indexOf(key);
if (index > -1) {
latestKeys.splice(index, 1);
}
}
}
}
export const isYAxisKey = (d: DataKey, limit: ValueSourceConfig): boolean => {
return (d.type === DataKeyType.function && d.label === limit.latestKey) ||
(d.type !== DataKeyType.function && d.name === limit.latestKey &&
d.type === limit.latestKeyType);
}
export const normalizeAxisLimit = (limit: string | number | ValueSourceConfig): ValueSourceConfig => {
if (!limit) {
return {
type: ValueSourceType.constant,
value: null,
entityAlias: null
};
} else if (typeof limit === 'number' || typeof limit === 'string') {
return {
type: ValueSourceType.constant,
value: Number(limit),
entityAlias: null
};
}
return limit;
}

23
ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts

@ -23,7 +23,7 @@ import {
createTimeSeriesYAxis,
defaultTimeSeriesChartYAxisSettings,
generateChartData,
LineSeriesStepType,
LineSeriesStepType, normalizeAxisLimit,
parseThresholdData,
TimeSeriesChartAxis,
TimeSeriesChartDataItem,
@ -581,8 +581,8 @@ export class TbTimeSeriesChart {
yAxisSettingsList.sort((a1, a2) => a1.order - a2.order);
const axisLimitDatasources: Datasource[] = [];
for (const yAxisSettings of yAxisSettingsList) {
yAxisSettings.min = this.normalizeAxisLimit(yAxisSettings.min);
yAxisSettings.max = this.normalizeAxisLimit(yAxisSettings.max);
yAxisSettings.min = normalizeAxisLimit(yAxisSettings.min);
yAxisSettings.max = normalizeAxisLimit(yAxisSettings.max);
const axisSettings = mergeDeep<TimeSeriesChartYAxisSettings>({} as TimeSeriesChartYAxisSettings,
defaultTimeSeriesChartYAxisSettings, yAxisSettings);
const units = isNotEmptyTbUnits(axisSettings.units) ? axisSettings.units : this.ctx.units;
@ -1080,21 +1080,4 @@ export class TbTimeSeriesChart {
this.timeSeriesChart.setOption(this.timeSeriesChartOptions);
}
}
private normalizeAxisLimit(limit: string | number | ValueSourceConfig): string | number | ValueSourceConfig {
if (!limit) {
return {
type: ValueSourceType.constant,
value: null,
entityAlias: null
};
} else if (typeof limit === 'number' || typeof limit === 'string') {
return {
type: ValueSourceType.constant,
value: Number(limit),
entityAlias: null
};
}
return limit;
}
}

11
ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.html

@ -24,8 +24,15 @@
<tb-toggle-option value="last">{{ 'widgets.recent-dashboards.last' | translate }}</tb-toggle-option>
<tb-toggle-option value="starred">{{ 'widgets.recent-dashboards.starred' | translate }}</tb-toggle-option>
</tb-toggle-header>
<a *ngIf="authUser.authority === authority.TENANT_ADMIN" class="md:!hidden"
mat-flat-button color="primary" routerLink="/dashboards" [queryParams]="{action: 'add'}">{{ 'dashboard.add' | translate }}</a>
<ng-container *ngIf="authUser.authority === authority.TENANT_ADMIN">
@if (hasDevice) {
<a class="md:!hidden"
mat-flat-button color="primary" routerLink="/dashboards/all" [queryParams]="{action: 'add'}">{{ 'dashboard.add' | translate }}</a>
} @else {
<a class="md:!hidden"
mat-stroked-button color="primary" routerLink="/dashboards/all" [queryParams]="{action: 'add'}">{{ 'dashboard.add' | translate }}</a>
}
</ng-container>
</div>
</div>
<ng-container *ngIf="userDashboardsInfo; else loading" [ngSwitch]="dashboardsToggle.value">

45
ui-ngx/src/app/modules/home/components/widget/lib/home-page/recent-dashboards-widget.component.ts

@ -48,6 +48,13 @@ import { Direction, SortOrder } from '@shared/models/page/sort-order';
import { MatSort } from '@angular/material/sort';
import { DashboardInfo } from '@shared/models/dashboard.models';
import { DashboardAutocompleteComponent } from '@shared/components/dashboard-autocomplete.component';
import { UtilsService } from '@core/services/utils.service';
import { Datasource, DatasourceType, widgetType } from '@shared/models/widget.models';
import { IWidgetSubscription, WidgetSubscriptionOptions } from '@core/api/widget-api.models';
import { formattedDataFormDatasourceData } from '@core/utils';
import { AliasFilterType } from '@shared/models/alias.models';
import { DataKeyType } from '@shared/models/telemetry/telemetry.models';
import { EntityType } from '@shared/models/entity-type.models';
@Component({
selector: 'tb-recent-dashboards-widget',
@ -78,13 +85,16 @@ export class RecentDashboardsWidgetComponent extends PageComponent implements On
starredDashboardValue = null;
hasDashboardsAccess = true;
hasDevice = true;
dirty = false;
public customerId: string;
private isFullscreenMode = getCurrentAuthState(this.store).forceFullscreen;
private subscription: IWidgetSubscription;
constructor(protected store: Store<AppState>,
private cd: ChangeDetectorRef,
private utils: UtilsService,
private userSettingService: UserSettingsService) {
super(store);
}
@ -96,6 +106,41 @@ export class RecentDashboardsWidgetComponent extends PageComponent implements On
this.hasDashboardsAccess = [Authority.TENANT_ADMIN, Authority.CUSTOMER_USER].includes(this.authUser.authority);
if (this.hasDashboardsAccess) {
this.reload();
if (window.location.pathname.startsWith('/home') && this.authUser.authority === Authority.TENANT_ADMIN) {
const ds: Datasource = {
type: DatasourceType.entityCount,
name: '',
entityFilter: {
entityType: EntityType.DEVICE,
type: AliasFilterType.entityType
},
dataKeys: [this.utils.createKey({ name: 'count'}, DataKeyType.count)]
}
const apiUsageSubscriptionOptions: WidgetSubscriptionOptions = {
datasources: [ds],
useDashboardTimewindow: false,
type: widgetType.latest,
callbacks: {
onDataUpdated: (subscription) => {
const data = formattedDataFormDatasourceData(subscription.data);
this.hasDevice = (data[0].count || 0) !== 0;
this.cd.detectChanges();
}
}
};
this.ctx.subscriptionApi.createSubscription(apiUsageSubscriptionOptions, true).subscribe((subscription) => {
this.subscription = subscription;
});
}
}
}
ngOnDestroy() {
super.ngOnDestroy();
if (this.subscription) {
this.ctx.subscriptionApi.removeSubscription(this.subscription.id);
}
}

40
ui-ngx/src/app/modules/home/components/widget/lib/rpc/power-button-widget.models.ts

@ -337,6 +337,9 @@ export abstract class PowerButtonShape {
protected onPowerSymbolLine: Path;
private onIcon$: Observable<Element>;
private offIcon$: Observable<Element>;
protected onIconOffsetX: number = 0;
protected onIconOffsetY: number = 0;
protected constructor(protected widgetContext: WidgetContext,
protected svgShape: Svg,
@ -362,10 +365,14 @@ export abstract class PowerButtonShape {
take(1),
map((svgElement) => {
const element = new Element(svgElement.firstChild);
const iconGroup = this.svgShape.group();
element.addTo(iconGroup);
const box = element.bbox();
const scale = size / box.height;
element.scale(scale);
return element;
const scaledBox = iconGroup.bbox();
iconGroup.translate(-scaledBox.cx, -scaledBox.cy);
return iconGroup;
}),
catchError(() => of(null)
));
@ -416,8 +423,15 @@ export abstract class PowerButtonShape {
public drawOffShape(centerGroup: G, label: boolean, labelWeight?: string, circleStroke?: boolean) {
if (this.icons.offButtonIcon.showIcon) {
this.createIconElement(this.icons.offButtonIcon.icon, this.icons.offButtonIcon.iconSize).subscribe(icon =>
this.offPowerSymbolIcon = icon.center(cx, cy).addTo(centerGroup));
this.offIcon$ = this.createIconElement(this.icons.offButtonIcon.icon, this.icons.offButtonIcon.iconSize)
.pipe(shareReplay(1));
this.offIcon$.pipe(take(1)).subscribe(icon => {
this.offPowerSymbolIcon = icon;
icon.translate(cx, cy);
centerGroup.add(icon);
});
} else {
if (label) {
this.offLabelShape = this.createOffLabel(labelWeight).addTo(centerGroup);
@ -434,9 +448,12 @@ export abstract class PowerButtonShape {
.pipe(shareReplay(1));
this.onIcon$.subscribe(icon => {
this.onPowerSymbolIcon = icon.center(cx, cy);
const iconBox = icon.bbox();
this.onIconOffsetX = iconBox.cx;
this.onIconOffsetY = iconBox.cy;
this.onPowerSymbolIcon = icon.translate(cx, cy);
if (isDefinedAndNotNull(onCenterGroup)) {
this.onPowerSymbolIcon.addTo(onCenterGroup);
onCenterGroup.add(this.onPowerSymbolIcon);
}
if (isDefinedAndNotNull(mask)) {
this.createMask(mask, [this.onPowerSymbolIcon]);
@ -468,7 +485,7 @@ export abstract class PowerButtonShape {
public onCenterTimeLine(timeline: Timeline, label: boolean) {
if (this.icons.onButtonIcon.showIcon) {
if (this.onIcon$) {
this.onIcon$.subscribe(icon => icon.timeline(timeline))
this.onIcon$.subscribe(icon => icon.timeline(timeline));
}
} else {
if (label) {
@ -482,7 +499,7 @@ export abstract class PowerButtonShape {
public offCenterColor(mainColor: PowerButtonColor, label: boolean) {
if (this.icons.offButtonIcon.showIcon) {
this.offPowerSymbolIcon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity});
this.offIcon$.subscribe(icon => icon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}))
} else {
if (label) {
this.offLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity});
@ -495,7 +512,7 @@ export abstract class PowerButtonShape {
public onCenterColor(mainColor: PowerButtonColor, label: boolean) {
if (this.icons.onButtonIcon.showIcon) {
this.onPowerSymbolIcon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity});
this.onIcon$.subscribe((icon)=> icon.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity}))
} else {
if (label) {
this.onLabelShape.attr({ fill: mainColor.hex, 'fill-opacity': mainColor.opacity});
@ -508,7 +525,12 @@ export abstract class PowerButtonShape {
public buttonAnimation(scale: number, label: boolean) {
if (this.icons.onButtonIcon.showIcon) {
powerButtonAnimation(this.onPowerSymbolIcon).transform({scale});
const translateX = cx - this.onIconOffsetX;
const translateY = cy - this.onIconOffsetY;
powerButtonAnimation(this.onPowerSymbolIcon).transform({
scale: scale,
translate: [translateX, translateY]
});
} else {
if (label) {
powerButtonAnimation(this.onLabelShape).transform({scale, origin: {x: cx, y: cy}});

6
ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.ts

@ -31,6 +31,7 @@ import {
barChartWithLabelsDefaultSettings
} from '@home/components/widget/lib/chart/bar-chart-with-labels-widget.models';
import { getSourceTbUnitSymbol } from '@shared/models/unit.models';
import { updateLatestDataKeys } from '@home/components/widget/lib/chart/time-series-chart.models';
@Component({
selector: 'tb-bar-chart-with-labels-widget-settings',
@ -123,6 +124,11 @@ export class BarChartWithLabelsWidgetSettingsComponent extends WidgetSettingsCom
});
}
protected onSettingsChanged(updated: WidgetSettings) {
updateLatestDataKeys([updated.yAxis], this.datasource, this.dataKeyCallbacks);
super.onSettingsChanged(updated);
}
protected validatorTriggers(): string[] {
return ['showBarLabel', 'showBarValue', 'showBarBorder', 'showLegend', 'showTooltip', 'tooltipShowDate'];
}

7
ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts

@ -30,7 +30,7 @@ import { rangeChartDefaultSettings } from '@home/components/widget/lib/chart/ran
import { DateFormatProcessor, DateFormatSettings } from '@shared/models/widget-settings.models';
import {
lineSeriesStepTypes,
lineSeriesStepTypeTranslations
lineSeriesStepTypeTranslations, updateLatestDataKeys
} from '@home/components/widget/lib/chart/time-series-chart.models';
import {
chartLabelPositions,
@ -269,6 +269,11 @@ export class RangeChartWidgetSettingsComponent extends WidgetSettingsComponent {
}
}
protected onSettingsChanged(updated: WidgetSettings) {
updateLatestDataKeys([updated.yAxis], this.datasource, this.dataKeyCallbacks);
super.onSettingsChanged(updated);
}
private _pointLabelPreviewFn(): string {
const units = getSourceTbUnitSymbol(this.widgetConfig.config.units);
const decimals: number = this.widgetConfig.config.decimals;

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save